You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by bl...@apache.org on 2025/05/05 18:09:19 UTC
(parquet-java) branch master updated: GH-3201: Implement a Variant builder to create Variant values (#3202)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-java.git
The following commit(s) were added to refs/heads/master by this push:
new 142bff02b GH-3201: Implement a Variant builder to create Variant values (#3202)
142bff02b is described below
commit 142bff02b09c468783f11f452b3dec9174c56a2a
Author: Gene Pang <77...@users.noreply.github.com>
AuthorDate: Mon May 5 11:09:15 2025 -0700
GH-3201: Implement a Variant builder to create Variant values (#3202)
---
.../parquet/variant/VariantArrayBuilder.java | 68 +++
.../org/apache/parquet/variant/VariantBuilder.java | 631 +++++++++++++++++++++
.../parquet/variant/VariantObjectBuilder.java | 91 +++
.../org/apache/parquet/variant/VariantUtil.java | 37 ++
.../apache/parquet/variant/TestVariantArray.java | 153 ++---
.../parquet/variant/TestVariantArrayBuilder.java | 293 ++++++++++
.../apache/parquet/variant/TestVariantObject.java | 197 +++----
.../parquet/variant/TestVariantObjectBuilder.java | 450 +++++++++++++++
.../apache/parquet/variant/TestVariantScalar.java | 453 ++++++++-------
.../parquet/variant/TestVariantScalarBuilder.java | 430 ++++++++++++++
.../apache/parquet/variant/VariantTestUtil.java | 106 ++++
11 files changed, 2459 insertions(+), 450 deletions(-)
diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantArrayBuilder.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantArrayBuilder.java
new file mode 100644
index 000000000..8dbd63318
--- /dev/null
+++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantArrayBuilder.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import java.util.ArrayList;
+
+/**
+ * Builder for creating Variant arrays, used by VariantBuilder.
+ */
+public class VariantArrayBuilder extends VariantBuilder {
+ /** The parent VariantBuilder. */
+ private final VariantBuilder parent;
+ /** The offsets of the elements in this array. */
+ private final ArrayList<Integer> offsets;
+ /** The number of values appended to this array. */
+ protected long numValues = 0;
+
+ VariantArrayBuilder(VariantBuilder parent) {
+ this.parent = parent;
+ this.offsets = new ArrayList<>();
+ }
+
+ /**
+ * @return the list of element offsets in this array
+ */
+ ArrayList<Integer> validateAndGetOffsets() {
+ if (offsets.size() != numValues) {
+ throw new IllegalStateException(String.format(
+ "Number of offsets (%d) do not match the number of values (%d).", offsets.size(), numValues));
+ }
+ checkMultipleNested("Cannot call endArray() while a nested object/array is still open.");
+ return offsets;
+ }
+
+ @Override
+ protected void onAppend() {
+ checkAppendWhileNested();
+ offsets.add(writePos);
+ numValues++;
+ }
+
+ @Override
+ protected void onStartNested() {
+ checkMultipleNested("Cannot call startObject()/startArray() without calling endObject()/endArray() first.");
+ offsets.add(writePos);
+ numValues++;
+ }
+
+ @Override
+ int addDictionaryKey(String key) {
+ // Add to the parent dictionary.
+ return parent.addDictionaryKey(key);
+ }
+}
diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java
new file mode 100644
index 000000000..47f7f8c56
--- /dev/null
+++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java
@@ -0,0 +1,631 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+
+/**
+ * Builder for creating Variant value and metadata.
+ */
+public class VariantBuilder {
+ /** The buffer for building the Variant value. The first `writePos` bytes have been written. */
+ protected byte[] writeBuffer = new byte[1024];
+
+ protected int writePos = 0;
+ /** The dictionary for mapping keys to monotonically increasing ids. */
+ private final HashMap<String, Integer> dictionary = new HashMap<>();
+ /** The keys in the dictionary, in id order. */
+ private final ArrayList<byte[]> dictionaryKeys = new ArrayList<>();
+
+ /**
+ * These are used to build nested objects and arrays, via startObject() and startArray().
+ * Only one of these can be non-null at a time. If one of these is non-null, then no append()
+ * methods can be called on this builder, until endObject() or endArray() is called.
+ */
+ protected VariantObjectBuilder objectBuilder = null;
+
+ protected VariantArrayBuilder arrayBuilder = null;
+
+ /**
+ * Creates a VariantBuilder.
+ */
+ public VariantBuilder() {}
+
+ /**
+ * @return the Variant value
+ */
+ public Variant build() {
+ if (objectBuilder != null) {
+ throw new IllegalStateException(
+ "Cannot call build() while an object is being built. Must call endObject() first.");
+ }
+ if (arrayBuilder != null) {
+ throw new IllegalStateException(
+ "Cannot call build() while an array is being built. Must call endArray() first.");
+ }
+ int numKeys = dictionaryKeys.size();
+ // Use long to avoid overflow in accumulating lengths.
+ long dictionaryTotalDataSize = 0;
+ for (byte[] key : dictionaryKeys) {
+ dictionaryTotalDataSize += key.length;
+ }
+ // Determine the number of bytes required per offset entry.
+ // The largest offset is the one-past-the-end value, which is total data size. It's very
+ // unlikely that the number of keys could be larger, but incorporate that into the calculation
+ // in case of pathological data.
+ long maxSize = Math.max(dictionaryTotalDataSize, numKeys);
+ int offsetSize = getMinIntegerSize((int) maxSize);
+
+ int offsetListOffset = 1 + offsetSize;
+ int dataOffset = offsetListOffset + (numKeys + 1) * offsetSize;
+ long metadataSize = dataOffset + dictionaryTotalDataSize;
+
+ byte[] metadata = new byte[(int) metadataSize];
+ // Only unsorted dictionary keys are supported.
+ // TODO: Support sorted dictionary keys.
+ int headerByte = VariantUtil.VERSION | ((offsetSize - 1) << 6);
+ VariantUtil.writeLong(metadata, 0, headerByte, 1);
+ VariantUtil.writeLong(metadata, 1, numKeys, offsetSize);
+ int currentOffset = 0;
+ for (int i = 0; i < numKeys; ++i) {
+ VariantUtil.writeLong(metadata, offsetListOffset + i * offsetSize, currentOffset, offsetSize);
+ byte[] key = dictionaryKeys.get(i);
+ System.arraycopy(key, 0, metadata, dataOffset + currentOffset, key.length);
+ currentOffset += key.length;
+ }
+ VariantUtil.writeLong(metadata, offsetListOffset + numKeys * offsetSize, currentOffset, offsetSize);
+ // Copying the data to a new buffer, to retain only the required data length, not the capacity.
+ // TODO: Reduce the copying, and look into builder reuse.
+ return new Variant(Arrays.copyOfRange(writeBuffer, 0, writePos), metadata);
+ }
+
+ /**
+ * Appends a string value to the Variant builder.
+ * @param str the string value to append
+ */
+ public void appendString(String str) {
+ onAppend();
+ byte[] data = str.getBytes(StandardCharsets.UTF_8);
+ boolean longStr = data.length > VariantUtil.MAX_SHORT_STR_SIZE;
+ checkCapacity((longStr ? 1 + VariantUtil.U32_SIZE : 1) + data.length);
+ if (longStr) {
+ writeBuffer[writePos] = VariantUtil.HEADER_LONG_STRING;
+ writePos += 1;
+ VariantUtil.writeLong(writeBuffer, writePos, data.length, VariantUtil.U32_SIZE);
+ writePos += VariantUtil.U32_SIZE;
+ } else {
+ writeBuffer[writePos] = VariantUtil.shortStrHeader(data.length);
+ writePos += 1;
+ }
+ System.arraycopy(data, 0, writeBuffer, writePos, data.length);
+ writePos += data.length;
+ }
+
+ /**
+ * Appends a null value to the Variant builder.
+ */
+ public void appendNull() {
+ onAppend();
+ checkCapacity(1);
+ writeBuffer[writePos] = VariantUtil.HEADER_NULL;
+ writePos += 1;
+ }
+
+ /**
+ * Appends a boolean value to the Variant builder.
+ * @param b the boolean value to append
+ */
+ public void appendBoolean(boolean b) {
+ onAppend();
+ checkCapacity(1);
+ writeBuffer[writePos] = b ? VariantUtil.HEADER_TRUE : VariantUtil.HEADER_FALSE;
+ writePos += 1;
+ }
+
+ /**
+ * Appends a long value to the variant builder.
+ * @param l the long value to append
+ */
+ public void appendLong(long l) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 8);
+ writeBuffer[writePos] = VariantUtil.HEADER_INT64;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, l, 8);
+ writePos += 9;
+ }
+
+ /**
+ * Appends an int value to the variant builder.
+ * @param i the int to append
+ */
+ public void appendInt(int i) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 4);
+ writeBuffer[writePos] = VariantUtil.HEADER_INT32;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, i, 4);
+ writePos += 5;
+ }
+
+ /**
+ * Appends a short value to the variant builder.
+ * @param s the short to append
+ */
+ public void appendShort(short s) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 2);
+ writeBuffer[writePos] = VariantUtil.HEADER_INT16;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, s, 2);
+ writePos += 3;
+ }
+
+ /**
+ * Appends a byte value to the variant builder.
+ * @param b the byte to append
+ */
+ public void appendByte(byte b) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 1);
+ writeBuffer[writePos] = VariantUtil.HEADER_INT8;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, b, 1);
+ writePos += 2;
+ }
+
+ /**
+ * Appends a double value to the variant builder.
+ * @param d the double to append
+ */
+ public void appendDouble(double d) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 8);
+ writeBuffer[writePos] = VariantUtil.HEADER_DOUBLE;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, Double.doubleToLongBits(d), 8);
+ writePos += 9;
+ }
+
+ /**
+ * Appends a decimal value to the variant builder. The actual encoded decimal type depends on the
+ * precision and scale of the decimal value.
+ * @param d the decimal value to append
+ */
+ public void appendDecimal(BigDecimal d) {
+ onAppend();
+ BigInteger unscaled = d.unscaledValue();
+ if (d.scale() <= VariantUtil.MAX_DECIMAL4_PRECISION && d.precision() <= VariantUtil.MAX_DECIMAL4_PRECISION) {
+ checkCapacity(2 /* header and scale size */ + 4);
+ writeBuffer[writePos] = VariantUtil.HEADER_DECIMAL4;
+ writeBuffer[writePos + 1] = (byte) d.scale();
+ VariantUtil.writeLong(writeBuffer, writePos + 2, unscaled.intValueExact(), 4);
+ writePos += 6;
+ } else if (d.scale() <= VariantUtil.MAX_DECIMAL8_PRECISION
+ && d.precision() <= VariantUtil.MAX_DECIMAL8_PRECISION) {
+ checkCapacity(2 /* header and scale size */ + 8);
+ writeBuffer[writePos] = VariantUtil.HEADER_DECIMAL8;
+ writeBuffer[writePos + 1] = (byte) d.scale();
+ VariantUtil.writeLong(writeBuffer, writePos + 2, unscaled.longValueExact(), 8);
+ writePos += 10;
+ } else {
+ assert d.scale() <= VariantUtil.MAX_DECIMAL16_PRECISION
+ && d.precision() <= VariantUtil.MAX_DECIMAL16_PRECISION;
+ checkCapacity(2 /* header and scale size */ + 16);
+ writeBuffer[writePos] = VariantUtil.HEADER_DECIMAL16;
+ writeBuffer[writePos + 1] = (byte) d.scale();
+ writePos += 2;
+ // `toByteArray` returns a big-endian representation. We need to copy it reversely and sign
+ // extend it to 16 bytes.
+ byte[] bytes = unscaled.toByteArray();
+ for (int i = 0; i < bytes.length; ++i) {
+ writeBuffer[writePos + i] = bytes[bytes.length - 1 - i];
+ }
+ byte sign = (byte) (bytes[0] < 0 ? -1 : 0);
+ for (int i = bytes.length; i < 16; ++i) {
+ writeBuffer[writePos + i] = sign;
+ }
+ writePos += 16;
+ }
+ }
+
+ /**
+ * Appends a date value to the variant builder. The date is represented as the number of days
+ * since the epoch.
+ * @param daysSinceEpoch the number of days since the epoch
+ */
+ public void appendDate(int daysSinceEpoch) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 4);
+ writeBuffer[writePos] = VariantUtil.HEADER_DATE;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, daysSinceEpoch, 4);
+ writePos += 5;
+ }
+
+ /**
+ * Appends a TimestampTz value to the variant builder. The timestamp is represented as the number
+ * of microseconds since the epoch.
+ * @param microsSinceEpoch the number of microseconds since the epoch
+ */
+ public void appendTimestampTz(long microsSinceEpoch) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 8);
+ writeBuffer[writePos] = VariantUtil.HEADER_TIMESTAMP_TZ;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, microsSinceEpoch, 8);
+ writePos += 9;
+ }
+
+ /**
+ * Appends a TimestampNtz value to the variant builder. The timestamp is represented as the number
+ * of microseconds since the epoch.
+ * @param microsSinceEpoch the number of microseconds since the epoch
+ */
+ public void appendTimestampNtz(long microsSinceEpoch) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 8);
+ writeBuffer[writePos] = VariantUtil.HEADER_TIMESTAMP_NTZ;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, microsSinceEpoch, 8);
+ writePos += 9;
+ }
+
+ /**
+ * Appends a Time value to the variant builder. The time is represented as the number of
+ * microseconds since midnight.
+ * @param microsSinceMidnight the number of microseconds since midnight
+ */
+ public void appendTime(long microsSinceMidnight) {
+ if (microsSinceMidnight < 0) {
+ throw new IllegalArgumentException(
+ String.format("Time value (%d) cannot be negative.", microsSinceMidnight));
+ }
+ onAppend();
+ checkCapacity(1 /* header size */ + 8);
+ writeBuffer[writePos] = VariantUtil.HEADER_TIME;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, microsSinceMidnight, 8);
+ writePos += 9;
+ }
+
+ /**
+ * Appends a TimestampNanosTz value to the variant builder. The timestamp is represented as the
+ * number of nanoseconds since the epoch.
+ * @param nanosSinceEpoch the number of nanoseconds since the epoch
+ */
+ public void appendTimestampNanosTz(long nanosSinceEpoch) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 8);
+ writeBuffer[writePos] = VariantUtil.HEADER_TIMESTAMP_NANOS_TZ;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, nanosSinceEpoch, 8);
+ writePos += 9;
+ }
+
+ /**
+ * Appends a TimestampNanosNtz value to the variant builder. The timestamp is represented as the
+ * number of nanoseconds since the epoch.
+ * @param nanosSinceEpoch the number of nanoseconds since the epoch
+ */
+ public void appendTimestampNanosNtz(long nanosSinceEpoch) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 8);
+ writeBuffer[writePos] = VariantUtil.HEADER_TIMESTAMP_NANOS_NTZ;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, nanosSinceEpoch, 8);
+ writePos += 9;
+ }
+
+ /**
+ * Appends a float value to the variant builder.
+ * @param f the float to append
+ */
+ public void appendFloat(float f) {
+ onAppend();
+ checkCapacity(1 /* header size */ + 4);
+ writeBuffer[writePos] = VariantUtil.HEADER_FLOAT;
+ VariantUtil.writeLong(writeBuffer, writePos + 1, Float.floatToIntBits(f), 8);
+ writePos += 5;
+ }
+
+ /**
+ * Appends binary data to the variant builder.
+ * @param binary the binary data to append
+ */
+ public void appendBinary(ByteBuffer binary) {
+ onAppend();
+ int binarySize = binary.remaining();
+ checkCapacity(1 /* header size */ + VariantUtil.U32_SIZE + binarySize);
+ writeBuffer[writePos] = VariantUtil.HEADER_BINARY;
+ writePos += 1;
+ VariantUtil.writeLong(writeBuffer, writePos, binarySize, VariantUtil.U32_SIZE);
+ writePos += VariantUtil.U32_SIZE;
+ ByteBuffer.wrap(writeBuffer, writePos, binarySize).put(binary);
+ writePos += binarySize;
+ }
+
+ /**
+ * Appends a UUID value to the variant builder.
+ * @param uuid the UUID to append
+ */
+ public void appendUUID(java.util.UUID uuid) {
+ onAppend();
+ checkCapacity(1 /* header size */ + VariantUtil.UUID_SIZE);
+ writeBuffer[writePos] = VariantUtil.HEADER_UUID;
+ writePos += 1;
+
+ ByteBuffer bb =
+ ByteBuffer.wrap(writeBuffer, writePos, VariantUtil.UUID_SIZE).order(ByteOrder.BIG_ENDIAN);
+ bb.putLong(uuid.getMostSignificantBits());
+ bb.putLong(uuid.getLeastSignificantBits());
+ writePos += VariantUtil.UUID_SIZE;
+ }
+
+ /**
+ * Starts appending an object to this variant builder. The returned VariantObjectBuilder is used
+ * to append object keys and values. startObject() must be called before endObject().
+ * No append*() methods can be called in between startObject() and endObject().
+ *
+ * Example usage:
+ * VariantBuilder builder = new VariantBuilder();
+ * VariantObjectBuilder objBuilder = builder.startObject();
+ * objBuilder.appendKey("key1");
+ * objBuilder.appendString("value1");
+ * builder.endObject();
+ *
+ * @return a VariantObjectBuilder to build an object
+ */
+ public VariantObjectBuilder startObject() {
+ onStartNested();
+ if (objectBuilder != null) {
+ throw new IllegalStateException("Cannot call startObject() without calling endObject() first.");
+ }
+ if (arrayBuilder != null) {
+ throw new IllegalStateException("Cannot call startObject() without calling endArray() first.");
+ }
+ this.objectBuilder = new VariantObjectBuilder(this);
+ return objectBuilder;
+ }
+
+ /**
+ * Finishes appending the object to this builder. This method must be called after startObject(),
+ * before other append*() methods can be called on this builder.
+ */
+ protected void endObject() {
+ if (objectBuilder == null) {
+ throw new IllegalStateException("Cannot call endObject() without calling startObject() first.");
+ }
+ ArrayList<FieldEntry> fields = objectBuilder.validateAndGetFields();
+ int numFields = fields.size();
+ Collections.sort(fields);
+ int maxId = numFields == 0 ? 0 : fields.get(0).id;
+ int dataSize = numFields == 0 ? 0 : fields.get(0).valueSize;
+
+ int distinctPos = 0;
+ // Maintain a list of distinct keys in-place.
+ for (int i = 1; i < numFields; ++i) {
+ maxId = Math.max(maxId, fields.get(i).id);
+ if (fields.get(i).id == fields.get(i - 1).id) {
+ // Found a duplicate key. Keep the field with the greater offset, because it was written last.
+ if (fields.get(distinctPos).offset < fields.get(i).offset) {
+ fields.set(distinctPos, fields.get(i));
+ }
+ } else {
+ // Found a distinct key. Add the field to the list.
+ distinctPos++;
+ fields.set(distinctPos, fields.get(i));
+ dataSize += fields.get(i).valueSize;
+ }
+ }
+
+ if (distinctPos + 1 < fields.size()) {
+ numFields = distinctPos + 1;
+ // Resize `fields` to `size`.
+ fields.subList(numFields, fields.size()).clear();
+ }
+
+ boolean largeSize = numFields > VariantUtil.U8_MAX;
+ int sizeBytes = largeSize ? VariantUtil.U32_SIZE : 1;
+ int idSize = getMinIntegerSize(maxId);
+ int offsetSize = getMinIntegerSize(dataSize);
+ // The data starts after: the header byte, object size, id list, and offset list.
+ int dataOffset = 1 + sizeBytes + numFields * idSize + (numFields + 1) * offsetSize;
+ checkCapacity(dataOffset + dataSize);
+
+ // Write the header byte and size entry.
+ writeBuffer[writePos] = VariantUtil.objectHeader(largeSize, idSize, offsetSize);
+ VariantUtil.writeLong(writeBuffer, writePos + 1, numFields, sizeBytes);
+
+ int idStart = writePos + 1 + sizeBytes;
+ int offsetStart = idStart + numFields * idSize;
+ int currOffset = 0;
+
+ // Loop over all fields and write the key id, offset, and data to the appropriate offsets
+ for (int i = 0; i < numFields; ++i) {
+ FieldEntry field = fields.get(i);
+ VariantUtil.writeLong(writeBuffer, idStart + i * idSize, field.id, idSize);
+ VariantUtil.writeLong(writeBuffer, offsetStart + i * offsetSize, currOffset, offsetSize);
+ System.arraycopy(
+ objectBuilder.writeBuffer,
+ field.offset,
+ writeBuffer,
+ writePos + dataOffset + currOffset,
+ field.valueSize);
+ currOffset += field.valueSize;
+ }
+ VariantUtil.writeLong(writeBuffer, offsetStart + numFields * offsetSize, dataSize, offsetSize);
+ writePos += dataOffset + dataSize;
+ this.objectBuilder = null;
+ }
+
+ /**
+ * Starts appending an array to this variant builder. The returned VariantArrayBuilder is used to
+ * append values ot the array. startArray() must be called before endArray(). No append*() methods
+ * can be called in between startArray() and endArray().
+ *
+ * Example usage:
+ * VariantBuilder builder = new VariantBuilder();
+ * VariantArrayBuilder arrayBuilder = builder.startArray();
+ * arrayBuilder.appendString("value1");
+ * arrayBuilder.appendString("value2");
+ * builder.endArray();
+ *
+ * @return a VariantArrayBuilder to use for startArrayElement() and endArray().
+ */
+ public VariantArrayBuilder startArray() {
+ onStartNested();
+ if (objectBuilder != null) {
+ throw new IllegalStateException("Cannot call startArray() without calling endObject() first.");
+ }
+ if (arrayBuilder != null) {
+ throw new IllegalStateException("Cannot call startArray() without calling endArray() first.");
+ }
+ this.arrayBuilder = new VariantArrayBuilder(this);
+ return arrayBuilder;
+ }
+
+ /**
+ * Ends appending an array to this variant builder. This method must be called after all elements
+ * have been added to the array.
+ */
+ public void endArray() {
+ if (arrayBuilder == null) {
+ throw new IllegalStateException("Cannot call endArray() without calling startArray() first.");
+ }
+ ArrayList<Integer> offsets = arrayBuilder.validateAndGetOffsets();
+ int numElements = offsets.size();
+ int dataSize = arrayBuilder.writePos;
+ boolean largeSize = numElements > VariantUtil.U8_MAX;
+ int sizeBytes = largeSize ? VariantUtil.U32_SIZE : 1;
+ int offsetSize = getMinIntegerSize(dataSize);
+ // The data starts after: the header byte, object size, and offset list.
+ int dataOffset = 1 + sizeBytes + (numElements + 1) * offsetSize;
+ checkCapacity(dataOffset + dataSize);
+
+ // Copy all the element data to the write buffer.
+ System.arraycopy(arrayBuilder.writeBuffer, 0, writeBuffer, writePos + dataOffset, dataSize);
+
+ writeBuffer[writePos] = VariantUtil.arrayHeader(largeSize, offsetSize);
+ VariantUtil.writeLong(writeBuffer, writePos + 1, numElements, sizeBytes);
+
+ int offsetStart = writePos + 1 + sizeBytes;
+ for (int i = 0; i < numElements; ++i) {
+ VariantUtil.writeLong(writeBuffer, offsetStart + i * offsetSize, offsets.get(i), offsetSize);
+ }
+ VariantUtil.writeLong(writeBuffer, offsetStart + numElements * offsetSize, dataSize, offsetSize);
+ writePos += dataOffset + dataSize;
+ this.arrayBuilder = null;
+ }
+
+ protected void onAppend() {
+ checkAppendWhileNested();
+ if (writePos > 0) {
+ throw new IllegalStateException("Cannot call multiple append() methods.");
+ }
+ }
+
+ protected void onStartNested() {
+ checkMultipleNested("Cannot call startObject()/startArray() without calling endObject()/endArray() first.");
+ }
+
+ protected void checkMultipleNested(String message) {
+ if (objectBuilder != null || arrayBuilder != null) {
+ throw new IllegalStateException(message);
+ }
+ }
+
+ protected void checkAppendWhileNested() {
+ if (objectBuilder != null) {
+ throw new IllegalStateException(
+ "Cannot call append() methods while an object is being built. Must call endObject() first.");
+ }
+ if (arrayBuilder != null) {
+ throw new IllegalStateException(
+ "Cannot call append() methods while an array is being built. Must call endArray() first.");
+ }
+ }
+
+ /**
+ * Adds a key to the Variant dictionary. If the key already exists, the dictionary is unmodified.
+ * @param key the key to add
+ * @return the id of the key
+ */
+ int addDictionaryKey(String key) {
+ return dictionary.computeIfAbsent(key, newKey -> {
+ int id = dictionaryKeys.size();
+ dictionaryKeys.add(newKey.getBytes(StandardCharsets.UTF_8));
+ return id;
+ });
+ }
+
+ /**
+ * @return the current write position of the variant builder
+ */
+ int writePos() {
+ return writePos;
+ }
+
+ /**
+ * Class to store the information of a Variant object field. We need to collect all fields of
+ * an object, sort them by their keys, and build the Variant object in sorted order.
+ */
+ static final class FieldEntry implements Comparable<FieldEntry> {
+ final String key;
+ final int id;
+ final int offset;
+ int valueSize = 0;
+
+ FieldEntry(String key, int id, int offset) {
+ this.key = key;
+ this.id = id;
+ this.offset = offset;
+ }
+
+ void updateValueSize(int size) {
+ valueSize = size;
+ }
+
+ @Override
+ public int compareTo(FieldEntry other) {
+ return key.compareTo(other.key);
+ }
+ }
+
+ private void checkCapacity(int additionalBytes) {
+ int requiredBytes = writePos + additionalBytes;
+ if (requiredBytes > writeBuffer.length) {
+ // Allocate a new buffer with a capacity of the next power of 2 of `requiredBytes`.
+ int newCapacity = Integer.highestOneBit(requiredBytes);
+ newCapacity = newCapacity < requiredBytes ? newCapacity * 2 : newCapacity;
+ byte[] newValue = new byte[newCapacity];
+ System.arraycopy(writeBuffer, 0, newValue, 0, writePos);
+ this.writeBuffer = newValue;
+ }
+ }
+
+ protected int getMinIntegerSize(int value) {
+ assert value >= 0;
+ if (value <= VariantUtil.U8_MAX) {
+ return VariantUtil.U8_SIZE;
+ }
+ if (value <= VariantUtil.U16_MAX) {
+ return VariantUtil.U16_SIZE;
+ }
+ if (value <= VariantUtil.U24_MAX) {
+ return VariantUtil.U24_SIZE;
+ }
+ return VariantUtil.U32_SIZE;
+ }
+}
diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantObjectBuilder.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantObjectBuilder.java
new file mode 100644
index 000000000..c10e0a01c
--- /dev/null
+++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantObjectBuilder.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import java.util.ArrayList;
+
+/**
+ * Builder for creating Variant object, used by VariantBuilder.
+ */
+public class VariantObjectBuilder extends VariantBuilder {
+ /** The parent VariantBuilder. */
+ private final VariantBuilder parent;
+ /** The FieldEntry list for the fields of this object. */
+ private final ArrayList<VariantBuilder.FieldEntry> fields;
+ /** The number of values appended to this object. */
+ protected long numValues = 0;
+
+ VariantObjectBuilder(VariantBuilder parent) {
+ this.parent = parent;
+ this.fields = new ArrayList<>();
+ }
+
+ /**
+ * Appends an object key to this object. This method must be called before appending any value.
+ * @param key the key to append
+ */
+ void appendKey(String key) {
+ if (fields.size() > numValues) {
+ throw new IllegalStateException("Cannot call appendKey() before appending a value for the previous key.");
+ }
+ updateLastValueSize();
+ fields.add(new VariantBuilder.FieldEntry(key, addDictionaryKey(key), writePos));
+ }
+
+ /**
+ * Returns the list of FieldEntry in this object. The state of the object is validated, so this
+ * object is guaranteed to have the same number of keys and values.
+ * @return the list of fields in this object
+ */
+ ArrayList<VariantBuilder.FieldEntry> validateAndGetFields() {
+ if (fields.size() != numValues) {
+ throw new IllegalStateException(String.format(
+ "Number of object keys (%d) do not match the number of values (%d).", fields.size(), numValues));
+ }
+ checkMultipleNested("Cannot call endObject() while a nested object/array is still open.");
+ updateLastValueSize();
+ return fields;
+ }
+
+ @Override
+ protected void onAppend() {
+ checkAppendWhileNested();
+ if (numValues != fields.size() - 1) {
+ throw new IllegalStateException("Cannot append an object value before calling appendKey()");
+ }
+ numValues++;
+ }
+
+ @Override
+ protected void onStartNested() {
+ checkMultipleNested("Cannot call startObject()/startArray() without calling endObject()/endArray() first.");
+ numValues++;
+ }
+
+ @Override
+ int addDictionaryKey(String key) {
+ // Add to the parent dictionary.
+ return parent.addDictionaryKey(key);
+ }
+
+ private void updateLastValueSize() {
+ if (!fields.isEmpty()) {
+ VariantBuilder.FieldEntry lastField = fields.get(fields.size() - 1);
+ lastField.updateValueSize(writePos() - lastField.offset);
+ }
+ }
+}
diff --git a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java
index 184cb7ec3..08fe185be 100644
--- a/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java
+++ b/parquet-variant/src/main/java/org/apache/parquet/variant/VariantUtil.java
@@ -187,6 +187,29 @@ class VariantUtil {
// The size (in bytes) of a UUID.
static final int UUID_SIZE = 16;
+ // header bytes
+ static final byte HEADER_NULL = primitiveHeader(NULL);
+ static final byte HEADER_LONG_STRING = primitiveHeader(LONG_STR);
+ static final byte HEADER_TRUE = primitiveHeader(TRUE);
+ static final byte HEADER_FALSE = primitiveHeader(FALSE);
+ static final byte HEADER_INT8 = primitiveHeader(INT8);
+ static final byte HEADER_INT16 = primitiveHeader(INT16);
+ static final byte HEADER_INT32 = primitiveHeader(INT32);
+ static final byte HEADER_INT64 = primitiveHeader(INT64);
+ static final byte HEADER_DOUBLE = primitiveHeader(DOUBLE);
+ static final byte HEADER_DECIMAL4 = primitiveHeader(DECIMAL4);
+ static final byte HEADER_DECIMAL8 = primitiveHeader(DECIMAL8);
+ static final byte HEADER_DECIMAL16 = primitiveHeader(DECIMAL16);
+ static final byte HEADER_DATE = primitiveHeader(DATE);
+ static final byte HEADER_TIMESTAMP_TZ = primitiveHeader(TIMESTAMP_TZ);
+ static final byte HEADER_TIMESTAMP_NTZ = primitiveHeader(TIMESTAMP_NTZ);
+ static final byte HEADER_TIME = primitiveHeader(TIME);
+ static final byte HEADER_TIMESTAMP_NANOS_TZ = primitiveHeader(TIMESTAMP_NANOS_TZ);
+ static final byte HEADER_TIMESTAMP_NANOS_NTZ = primitiveHeader(TIMESTAMP_NANOS_NTZ);
+ static final byte HEADER_FLOAT = primitiveHeader(FLOAT);
+ static final byte HEADER_BINARY = primitiveHeader(BINARY);
+ static final byte HEADER_UUID = primitiveHeader(UUID);
+
static byte primitiveHeader(int type) {
return (byte) (type << 2 | PRIMITIVE);
}
@@ -219,6 +242,20 @@ class VariantUtil {
}
}
+ /**
+ * Write the least significant `numBytes` bytes in `value` into `bytes[pos, pos + numBytes)` in
+ * little endian.
+ * @param bytes The byte array to write into
+ * @param pos The starting index of the byte array to write into
+ * @param value The value to write
+ * @param numBytes The number of bytes to write
+ */
+ static void writeLong(byte[] bytes, int pos, long value, int numBytes) {
+ for (int i = 0; i < numBytes; ++i) {
+ bytes[pos + i] = (byte) ((value >>> (8 * i)) & 0xFF);
+ }
+ }
+
/**
* Reads a little-endian signed long value from `buffer[pos, pos + numBytes)`.
* @param buffer The ByteBuffer to read from
diff --git a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantArray.java b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantArray.java
index cb465d1d2..6e92b8576 100644
--- a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantArray.java
+++ b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantArray.java
@@ -20,11 +20,7 @@ package org.apache.parquet.variant;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-import java.nio.charset.StandardCharsets;
-import java.security.SecureRandom;
import java.time.LocalDate;
-import java.util.*;
-import java.util.function.Consumer;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -32,84 +28,16 @@ import org.slf4j.LoggerFactory;
public class TestVariantArray {
private static final Logger LOG = LoggerFactory.getLogger(TestVariantArray.class);
- private static final String RANDOM_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
- /** Random number generator for generating random strings */
- private static SecureRandom random = new SecureRandom(new byte[] {1, 2, 3, 4, 5});
-
- private static final ByteBuffer EMPTY_METADATA = ByteBuffer.wrap(new byte[] {0b1});
-
- private static final byte[] VALUE_NULL = new byte[] {primitiveHeader(0)};
- private static final byte[] VALUE_BOOL = new byte[] {primitiveHeader(1)};
- private static final byte[] VALUE_INT = new byte[] {primitiveHeader(5), (byte) 0xD2, 0x02, (byte) 0x96, 0x49};
+ private static final byte[] VALUE_NULL = new byte[] {VariantTestUtil.primitiveHeader(0)};
+ private static final byte[] VALUE_BOOL = new byte[] {VariantTestUtil.primitiveHeader(1)};
+ private static final byte[] VALUE_INT =
+ new byte[] {VariantTestUtil.primitiveHeader(5), (byte) 0xD2, 0x02, (byte) 0x96, 0x49};
private static final byte[] VALUE_STRING =
- new byte[] {primitiveHeader(16), 0x07, 0x00, 0x00, 0x00, 'v', 'a', 'r', 'i', 'a', 'n', 't'};
+ new byte[] {VariantTestUtil.primitiveHeader(16), 0x07, 0x00, 0x00, 0x00, 'v', 'a', 'r', 'i', 'a', 'n', 't'};
private static final byte[] VALUE_SHORT_STRING = new byte[] {0b101, 'c'};
private static final byte[] VALUE_DATE = new byte[] {0b101100, (byte) 0xE3, 0x4E, 0x00, 0x00};
- private void checkType(Variant v, int expectedBasicType, Variant.Type expectedType) {
- Assert.assertEquals(expectedBasicType, v.value.get(v.value.position()) & VariantUtil.BASIC_TYPE_MASK);
- Assert.assertEquals(expectedType, v.getType());
- }
-
- private String randomString(int len) {
- StringBuilder sb = new StringBuilder(len);
- for (int i = 0; i < len; i++) {
- sb.append(RANDOM_CHARS.charAt(random.nextInt(RANDOM_CHARS.length())));
- }
- return sb.toString();
- }
-
- private void testVariant(Variant v, Consumer<Variant> consumer) {
- consumer.accept(v);
- // Create new Variant with different byte offsets
- byte[] newValue = new byte[v.value.capacity() + 50];
- byte[] newMetadata = new byte[v.metadata.capacity() + 50];
- Arrays.fill(newValue, (byte) 0xFF);
- Arrays.fill(newMetadata, (byte) 0xFF);
- v.value.position(0);
- v.value.get(newValue, 25, v.value.capacity());
- v.value.position(0);
- v.metadata.position(0);
- v.metadata.get(newMetadata, 25, v.metadata.capacity());
- v.metadata.position(0);
- Variant v2 = new Variant(
- ByteBuffer.wrap(newValue, 25, v.value.capacity()),
- ByteBuffer.wrap(newMetadata, 25, v.metadata.capacity()));
- consumer.accept(v2);
- }
-
- private static byte primitiveHeader(int type) {
- return (byte) (type << 2);
- }
-
- private static int getMinIntegerSize(int value) {
- return (value <= 0xFF) ? 1 : (value <= 0xFFFF) ? 2 : (value <= 0xFFFFFF) ? 3 : 4;
- }
-
- private static void writeVarlenInt(ByteBuffer buffer, int value, int valueSize) {
- if (valueSize == 1) {
- buffer.put((byte) value);
- } else if (valueSize == 2) {
- buffer.putShort((short) value);
- } else if (valueSize == 3) {
- buffer.put((byte) (value & 0xFF));
- buffer.put((byte) ((value >> 8) & 0xFF));
- buffer.put((byte) ((value >> 16) & 0xFF));
- } else {
- buffer.putInt(value);
- }
- }
-
- private static byte[] constructString(String value) {
- return ByteBuffer.allocate(value.length() + 5)
- .order(ByteOrder.LITTLE_ENDIAN)
- .put(primitiveHeader(16))
- .putInt(value.length())
- .put(value.getBytes(StandardCharsets.UTF_8))
- .array();
- }
-
private static byte[] constructArray(byte[]... elements) {
int dataSize = 0;
for (byte[] element : elements) {
@@ -117,7 +45,7 @@ public class TestVariantArray {
}
boolean isLarge = elements.length > 0xFF;
- int offsetSize = getMinIntegerSize(dataSize);
+ int offsetSize = VariantTestUtil.getMinIntegerSize(dataSize);
int headerSize = 1 + (isLarge ? 4 : 1) + (elements.length + 1) * offsetSize;
ByteBuffer output = ByteBuffer.allocate(headerSize + dataSize).order(ByteOrder.LITTLE_ENDIAN);
@@ -132,10 +60,10 @@ public class TestVariantArray {
int currOffset = 0;
for (int i = 0; i < elements.length; ++i) {
- writeVarlenInt(output, currOffset, offsetSize);
+ VariantTestUtil.writeVarlenInt(output, currOffset, offsetSize);
currOffset += elements[i].length;
}
- writeVarlenInt(output, currOffset, offsetSize);
+ VariantTestUtil.writeVarlenInt(output, currOffset, offsetSize);
for (int i = 0; i < elements.length; ++i) {
output.put(elements[i]);
@@ -146,18 +74,19 @@ public class TestVariantArray {
@Test
public void testEmptyArray() {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b0011, 0x00}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b0011, 0x00}), VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
Assert.assertEquals(0, v.numArrayElements());
});
}
@Test
public void testEmptyLargeArray() {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b10011, 0x00, 0x00, 0x00, 0x00}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {0b10011, 0x00, 0x00, 0x00, 0x00}), VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
Assert.assertEquals(0, v.numArrayElements());
});
}
@@ -165,9 +94,10 @@ public class TestVariantArray {
@Test
public void testLargeArraySize() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {0b10011, (byte) 0xFF, (byte) 0x01, 0x00, 0x00}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ ByteBuffer.wrap(new byte[] {0b10011, (byte) 0xFF, (byte) 0x01, 0x00, 0x00}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
Assert.assertEquals(511, v.numArrayElements());
});
}
@@ -177,45 +107,46 @@ public class TestVariantArray {
byte[] nested = constructArray(VALUE_INT, VALUE_NULL, VALUE_SHORT_STRING);
Variant value = new Variant(
ByteBuffer.wrap(constructArray(VALUE_DATE, VALUE_BOOL, VALUE_INT, VALUE_STRING, nested)),
- EMPTY_METADATA);
+ VariantTestUtil.EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
Assert.assertEquals(5, v.numArrayElements());
- checkType(v.getElementAtIndex(0), VariantUtil.PRIMITIVE, Variant.Type.DATE);
+ VariantTestUtil.checkType(v.getElementAtIndex(0), VariantUtil.PRIMITIVE, Variant.Type.DATE);
Assert.assertEquals(
LocalDate.parse("2025-04-17"),
LocalDate.ofEpochDay(v.getElementAtIndex(0).getInt()));
- checkType(v.getElementAtIndex(1), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ VariantTestUtil.checkType(v.getElementAtIndex(1), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
Assert.assertTrue(v.getElementAtIndex(1).getBoolean());
- checkType(v.getElementAtIndex(2), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ VariantTestUtil.checkType(v.getElementAtIndex(2), VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(1234567890, v.getElementAtIndex(2).getInt());
- checkType(v.getElementAtIndex(3), VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ VariantTestUtil.checkType(v.getElementAtIndex(3), VariantUtil.PRIMITIVE, Variant.Type.STRING);
Assert.assertEquals("variant", v.getElementAtIndex(3).getString());
- checkType(v.getElementAtIndex(4), VariantUtil.ARRAY, Variant.Type.ARRAY);
+ VariantTestUtil.checkType(v.getElementAtIndex(4), VariantUtil.ARRAY, Variant.Type.ARRAY);
Variant nestedV = v.getElementAtIndex(4);
Assert.assertEquals(3, nestedV.numArrayElements());
- checkType(nestedV.getElementAtIndex(0), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ VariantTestUtil.checkType(nestedV.getElementAtIndex(0), VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(1234567890, nestedV.getElementAtIndex(0).getInt());
- checkType(nestedV.getElementAtIndex(1), VariantUtil.PRIMITIVE, Variant.Type.NULL);
- checkType(nestedV.getElementAtIndex(2), VariantUtil.SHORT_STR, Variant.Type.STRING);
+ VariantTestUtil.checkType(nestedV.getElementAtIndex(1), VariantUtil.PRIMITIVE, Variant.Type.NULL);
+ VariantTestUtil.checkType(nestedV.getElementAtIndex(2), VariantUtil.SHORT_STR, Variant.Type.STRING);
Assert.assertEquals("c", nestedV.getElementAtIndex(2).getString());
});
}
- public void testArrayOffsetSize(String randomString) {
+ private void testArrayOffsetSize(String randomString) {
Variant value = new Variant(
- ByteBuffer.wrap(constructArray(constructString(randomString), VALUE_BOOL, VALUE_INT)), EMPTY_METADATA);
+ ByteBuffer.wrap(constructArray(VariantTestUtil.constructString(randomString), VALUE_BOOL, VALUE_INT)),
+ VariantTestUtil.EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
Assert.assertEquals(3, v.numArrayElements());
- checkType(v.getElementAtIndex(0), VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ VariantTestUtil.checkType(v.getElementAtIndex(0), VariantUtil.PRIMITIVE, Variant.Type.STRING);
Assert.assertEquals(randomString, v.getElementAtIndex(0).getString());
- checkType(v.getElementAtIndex(1), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ VariantTestUtil.checkType(v.getElementAtIndex(1), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
Assert.assertTrue(v.getElementAtIndex(1).getBoolean());
- checkType(v.getElementAtIndex(2), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ VariantTestUtil.checkType(v.getElementAtIndex(2), VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(1234567890, v.getElementAtIndex(2).getInt());
});
}
@@ -223,26 +154,26 @@ public class TestVariantArray {
@Test
public void testArrayTwoByteOffset() {
// a string larger than 255 bytes to push the value offset size above 1 byte
- testArrayOffsetSize(randomString(300));
+ testArrayOffsetSize(VariantTestUtil.randomString(300));
}
@Test
public void testArrayThreeByteOffset() {
// a string larger than 65535 bytes to push the value offset size above 2 bytes
- testArrayOffsetSize(randomString(70_000));
+ testArrayOffsetSize(VariantTestUtil.randomString(70_000));
}
@Test
public void testArrayFourByteOffset() {
// a string larger than 16777215 bytes to push the value offset size above 3 bytes
- testArrayOffsetSize(randomString(16_800_000));
+ testArrayOffsetSize(VariantTestUtil.randomString(16_800_000));
}
@Test
public void testInvalidArray() {
try {
// An object header
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b1000010}), EMPTY_METADATA);
+ Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b1000010}), VariantTestUtil.EMPTY_METADATA);
value.numArrayElements();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
diff --git a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantArrayBuilder.java b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantArrayBuilder.java
new file mode 100644
index 000000000..ccb4f7b1f
--- /dev/null
+++ b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantArrayBuilder.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestVariantArrayBuilder {
+ private static final Logger LOG = LoggerFactory.getLogger(TestVariantArrayBuilder.class);
+
+ @Test
+ public void testEmptyArrayBuilder() {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder a = b.startArray();
+ b.endArray();
+ VariantTestUtil.testVariant(b.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Assert.assertEquals(0, v.numArrayElements());
+ });
+ }
+
+ @Test
+ public void testLargeArraySizeBuilder() {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder a = b.startArray();
+ for (int i = 0; i < 511; i++) {
+ a.appendInt(i);
+ }
+ b.endArray();
+ VariantTestUtil.testVariant(b.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Assert.assertEquals(511, v.numArrayElements());
+ for (int i = 0; i < 511; i++) {
+ VariantTestUtil.checkType(v.getElementAtIndex(i), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ Assert.assertEquals(i, v.getElementAtIndex(i).getInt());
+ }
+ });
+ }
+
+ @Test
+ public void testMixedArrayBuilder() {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder arrBuilder = b.startArray();
+ arrBuilder.appendBoolean(true);
+ VariantObjectBuilder obj = arrBuilder.startObject();
+ obj.appendKey("key");
+ obj.appendInt(321);
+ arrBuilder.endObject();
+ arrBuilder.appendLong(1234567890);
+ {
+ // build a nested array
+ VariantArrayBuilder nestedBuilder = arrBuilder.startArray();
+ {
+ // build a nested empty array
+ nestedBuilder.startArray();
+ nestedBuilder.endArray();
+ }
+ nestedBuilder.appendString("variant");
+ nestedBuilder.startObject();
+ nestedBuilder.endObject();
+ arrBuilder.endArray();
+ }
+ b.endArray();
+
+ VariantTestUtil.testVariant(b.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Assert.assertEquals(4, v.numArrayElements());
+ VariantTestUtil.checkType(v.getElementAtIndex(0), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ Assert.assertTrue(v.getElementAtIndex(0).getBoolean());
+
+ VariantTestUtil.checkType(v.getElementAtIndex(1), VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Assert.assertEquals(1, v.getElementAtIndex(1).numObjectElements());
+ VariantTestUtil.checkType(
+ v.getElementAtIndex(1).getFieldByKey("key"), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ Assert.assertEquals(321, v.getElementAtIndex(1).getFieldByKey("key").getInt());
+
+ VariantTestUtil.checkType(v.getElementAtIndex(2), VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ Assert.assertEquals(1234567890, v.getElementAtIndex(2).getLong());
+
+ VariantTestUtil.checkType(v.getElementAtIndex(3), VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Variant nested = v.getElementAtIndex(3);
+ Assert.assertEquals(3, nested.numArrayElements());
+ VariantTestUtil.checkType(nested.getElementAtIndex(0), VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Assert.assertEquals(0, nested.getElementAtIndex(0).numArrayElements());
+ VariantTestUtil.checkType(nested.getElementAtIndex(1), VariantUtil.SHORT_STR, Variant.Type.STRING);
+ Assert.assertEquals("variant", nested.getElementAtIndex(1).getString());
+ VariantTestUtil.checkType(nested.getElementAtIndex(2), VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Assert.assertEquals(0, nested.getElementAtIndex(2).numObjectElements());
+ });
+ }
+
+ private void buildNested(int i, VariantArrayBuilder obj) {
+ if (i > 0) {
+ obj.appendString("str" + i);
+ buildNested(i - 1, obj.startArray());
+ obj.endArray();
+ }
+ }
+
+ @Test
+ public void testNestedBuilder() {
+ VariantBuilder b = new VariantBuilder();
+ buildNested(1000, b.startArray());
+ b.endArray();
+
+ VariantTestUtil.testVariant(b.build(), v -> {
+ Variant curr = v;
+ for (int i = 1000; i >= 0; i--) {
+ VariantTestUtil.checkType(curr, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ if (i == 0) {
+ Assert.assertEquals(0, curr.numArrayElements());
+ } else {
+ Assert.assertEquals(2, curr.numArrayElements());
+ VariantTestUtil.checkType(curr.getElementAtIndex(0), VariantUtil.SHORT_STR, Variant.Type.STRING);
+ Assert.assertEquals("str" + i, curr.getElementAtIndex(0).getString());
+ curr = curr.getElementAtIndex(1);
+ }
+ }
+ });
+ }
+
+ private void testArrayOffsetSizeBuilder(String randomString) {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder arrBuilder = b.startArray();
+ arrBuilder.appendString(randomString);
+ arrBuilder.appendBoolean(true);
+ arrBuilder.appendLong(1234567890);
+ b.endArray();
+
+ VariantTestUtil.testVariant(b.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Assert.assertEquals(3, v.numArrayElements());
+ VariantTestUtil.checkType(v.getElementAtIndex(0), VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ Assert.assertEquals(randomString, v.getElementAtIndex(0).getString());
+ VariantTestUtil.checkType(v.getElementAtIndex(1), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ Assert.assertTrue(v.getElementAtIndex(1).getBoolean());
+ VariantTestUtil.checkType(v.getElementAtIndex(2), VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ Assert.assertEquals(1234567890, v.getElementAtIndex(2).getLong());
+ });
+ }
+
+ @Test
+ public void testArrayTwoByteOffsetBuilder() {
+ // a string larger than 255 bytes to push the value offset size above 1 byte
+ testArrayOffsetSizeBuilder(VariantTestUtil.randomString(300));
+ }
+
+ @Test
+ public void testArrayThreeByteOffsetBuilder() {
+ // a string larger than 65535 bytes to push the value offset size above 2 bytes
+ testArrayOffsetSizeBuilder(VariantTestUtil.randomString(70_000));
+ }
+
+ @Test
+ public void testArrayFourByteOffsetBuilder() {
+ // a string larger than 16777215 bytes to push the value offset size above 3 bytes
+ testArrayOffsetSizeBuilder(VariantTestUtil.randomString(16_800_000));
+ }
+
+ @Test
+ public void testMissingEndArray() {
+ VariantBuilder b = new VariantBuilder();
+ b.startArray();
+ try {
+ b.build();
+ Assert.fail("Expected Exception when calling build() without endArray()");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testMissingStartArray() {
+ VariantBuilder b = new VariantBuilder();
+ try {
+ b.endArray();
+ Assert.fail("Expected Exception when calling endArray() without startArray()");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testInvalidAppendDuringArray() {
+ VariantBuilder b = new VariantBuilder();
+ b.startArray();
+ try {
+ b.appendInt(1);
+ Assert.fail("Expected Exception when calling append() before endArray()");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testStartArrayEndObject() {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder obj = b.startArray();
+ try {
+ obj.endObject();
+ Assert.fail("Expected Exception when calling endObject() while building array");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedObject() {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder arr = b.startArray();
+ arr.startObject();
+ try {
+ b.endArray();
+ Assert.fail("Expected Exception when calling endArray() with an open nested object");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedObjectWithKey() {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder arr = b.startArray();
+ VariantObjectBuilder nested = arr.startObject();
+ nested.appendKey("nested");
+ try {
+ b.endArray();
+ Assert.fail("Expected Exception when calling endArray() with an open nested object");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedObjectWithKeyValue() {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder arr = b.startArray();
+ VariantObjectBuilder nested = arr.startObject();
+ nested.appendKey("nested");
+ nested.appendInt(1);
+ try {
+ b.endArray();
+ Assert.fail("Expected Exception when calling endArray() with an open nested object");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedArray() {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder arr = b.startArray();
+ arr.startArray();
+ try {
+ b.endArray();
+ Assert.fail("Expected Exception when calling endArray() with an open nested array");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedArrayWithElement() {
+ VariantBuilder b = new VariantBuilder();
+ VariantArrayBuilder arr = b.startArray();
+ VariantArrayBuilder nested = arr.startArray();
+ nested.appendInt(1);
+ try {
+ b.endArray();
+ Assert.fail("Expected Exception when calling endArray() with an open nested array");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+}
diff --git a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObject.java b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObject.java
index 12f5a334c..22a53976e 100644
--- a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObject.java
+++ b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObject.java
@@ -23,10 +23,13 @@ import com.google.common.collect.ImmutableMap;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
-import java.security.SecureRandom;
import java.time.LocalDate;
-import java.util.*;
-import java.util.function.Consumer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -34,87 +37,15 @@ import org.slf4j.LoggerFactory;
public class TestVariantObject {
private static final Logger LOG = LoggerFactory.getLogger(TestVariantObject.class);
- private static final String RANDOM_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
- /** Random number generator for generating random strings */
- private static SecureRandom random = new SecureRandom(new byte[] {1, 2, 3, 4, 5});
-
- private static final ByteBuffer EMPTY_METADATA = ByteBuffer.wrap(new byte[] {0b1});
-
- private static final byte[] VALUE_NULL = new byte[] {primitiveHeader(0)};
- private static final byte[] VALUE_BOOL = new byte[] {primitiveHeader(1)};
- private static final byte[] VALUE_INT = new byte[] {primitiveHeader(5), (byte) 0xD2, 0x02, (byte) 0x96, 0x49};
+ private static final byte[] VALUE_NULL = new byte[] {VariantTestUtil.primitiveHeader(0)};
+ private static final byte[] VALUE_BOOL = new byte[] {VariantTestUtil.primitiveHeader(1)};
+ private static final byte[] VALUE_INT =
+ new byte[] {VariantTestUtil.primitiveHeader(5), (byte) 0xD2, 0x02, (byte) 0x96, 0x49};
private static final byte[] VALUE_STRING =
- new byte[] {primitiveHeader(16), 0x07, 0x00, 0x00, 0x00, 'v', 'a', 'r', 'i', 'a', 'n', 't'};
+ new byte[] {VariantTestUtil.primitiveHeader(16), 0x07, 0x00, 0x00, 0x00, 'v', 'a', 'r', 'i', 'a', 'n', 't'};
private static final byte[] VALUE_DATE = new byte[] {0b101100, (byte) 0xE3, 0x4E, 0x00, 0x00};
- private void checkType(Variant v, int expectedBasicType, Variant.Type expectedType) {
- Assert.assertEquals(expectedBasicType, v.value.get(v.value.position()) & VariantUtil.BASIC_TYPE_MASK);
- Assert.assertEquals(expectedType, v.getType());
- }
-
- private String randomString(int len) {
- StringBuilder sb = new StringBuilder(len);
- for (int i = 0; i < len; i++) {
- sb.append(RANDOM_CHARS.charAt(random.nextInt(RANDOM_CHARS.length())));
- }
- return sb.toString();
- }
-
- private void testVariant(Variant v, Consumer<Variant> consumer) {
- consumer.accept(v);
- // Create new Variant with different byte offsets
- byte[] newValue = new byte[v.value.capacity() + 50];
- byte[] newMetadata = new byte[v.metadata.capacity() + 50];
- Arrays.fill(newValue, (byte) 0xFF);
- Arrays.fill(newMetadata, (byte) 0xFF);
- v.value.position(0);
- v.value.get(newValue, 25, v.value.capacity());
- v.value.position(0);
- v.metadata.position(0);
- v.metadata.get(newMetadata, 25, v.metadata.capacity());
- v.metadata.position(0);
- Variant v2 = new Variant(
- ByteBuffer.wrap(newValue, 25, v.value.capacity()),
- ByteBuffer.wrap(newMetadata, 25, v.metadata.capacity()));
- consumer.accept(v2);
- }
-
- private static byte primitiveHeader(int type) {
- return (byte) (type << 2);
- }
-
- private static byte metadataHeader(boolean isSorted, int offsetSize) {
- return (byte) (((offsetSize - 1) << 6) | (isSorted ? 0b10000 : 0) | 0b0001);
- }
-
- private static int getMinIntegerSize(int value) {
- return (value <= 0xFF) ? 1 : (value <= 0xFFFF) ? 2 : (value <= 0xFFFFFF) ? 3 : 4;
- }
-
- private static void writeVarlenInt(ByteBuffer buffer, int value, int valueSize) {
- if (valueSize == 1) {
- buffer.put((byte) value);
- } else if (valueSize == 2) {
- buffer.putShort((short) value);
- } else if (valueSize == 3) {
- buffer.put((byte) (value & 0xFF));
- buffer.put((byte) ((value >> 8) & 0xFF));
- buffer.put((byte) ((value >> 16) & 0xFF));
- } else {
- buffer.putInt(value);
- }
- }
-
- private static byte[] constructString(String value) {
- return ByteBuffer.allocate(value.length() + 5)
- .order(ByteOrder.LITTLE_ENDIAN)
- .put(primitiveHeader(16))
- .putInt(value.length())
- .put(value.getBytes(StandardCharsets.UTF_8))
- .array();
- }
-
private static byte[] constructObject(Map<String, Integer> keys, Map<String, byte[]> fields, boolean orderedData) {
int dataSize = 0;
int maxId = 0;
@@ -124,8 +55,8 @@ public class TestVariantObject {
}
boolean isLarge = fields.size() > 0xFF;
- int fieldIdSize = getMinIntegerSize(maxId);
- int offsetSize = getMinIntegerSize(dataSize);
+ int fieldIdSize = VariantTestUtil.getMinIntegerSize(maxId);
+ int offsetSize = VariantTestUtil.getMinIntegerSize(dataSize);
// The space for header byte, object size, id list, and offset list.
int headerSize = 1 + (isLarge ? 4 : 1) + fields.size() * fieldIdSize + (fields.size() + 1) * offsetSize;
@@ -145,17 +76,17 @@ public class TestVariantObject {
// write field ids
for (String fieldName : sortedFieldNames) {
int fieldId = keys.get(fieldName);
- writeVarlenInt(output, fieldId, fieldIdSize);
+ VariantTestUtil.writeVarlenInt(output, fieldId, fieldIdSize);
}
// write offsets
int currOffset = 0;
for (String fieldName : sortedFieldNames) {
int offsetToWrite = orderedData ? currOffset : dataSize - currOffset - fields.get(fieldName).length;
- writeVarlenInt(output, offsetToWrite, offsetSize);
+ VariantTestUtil.writeVarlenInt(output, offsetToWrite, offsetSize);
currOffset += fields.get(fieldName).length;
}
- writeVarlenInt(output, orderedData ? currOffset : 0, offsetSize);
+ VariantTestUtil.writeVarlenInt(output, orderedData ? currOffset : 0, offsetSize);
// write data
for (int i = 0; i < sortedFieldNames.length; ++i) {
@@ -168,7 +99,7 @@ public class TestVariantObject {
private static ByteBuffer constructMetadata(Boolean isSorted, List<String> fieldNames) {
if (fieldNames.isEmpty()) {
- return EMPTY_METADATA;
+ return VariantTestUtil.EMPTY_METADATA;
}
int dataSize = 0;
@@ -176,23 +107,23 @@ public class TestVariantObject {
dataSize += fieldName.length();
}
- int offsetSize = getMinIntegerSize(dataSize);
+ int offsetSize = VariantTestUtil.getMinIntegerSize(dataSize);
int offsetListStart = 1 + offsetSize;
int stringStart = offsetListStart + (fieldNames.size() + 1) * offsetSize;
int metadataSize = stringStart + dataSize;
ByteBuffer output = ByteBuffer.allocate(metadataSize).order(ByteOrder.LITTLE_ENDIAN);
- output.put(metadataHeader(isSorted, offsetSize));
- writeVarlenInt(output, fieldNames.size(), offsetSize);
+ output.put(VariantTestUtil.metadataHeader(isSorted, offsetSize));
+ VariantTestUtil.writeVarlenInt(output, fieldNames.size(), offsetSize);
// write offsets
int currentOffset = 0;
for (String fieldName : fieldNames) {
- writeVarlenInt(output, currentOffset, offsetSize);
+ VariantTestUtil.writeVarlenInt(output, currentOffset, offsetSize);
currentOffset += fieldName.length();
}
- writeVarlenInt(output, currentOffset, offsetSize);
+ VariantTestUtil.writeVarlenInt(output, currentOffset, offsetSize);
// write strings
for (String fieldName : fieldNames) {
@@ -205,18 +136,19 @@ public class TestVariantObject {
@Test
public void testEmptyObject() {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b10, 0x00}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b10, 0x00}), VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
Assert.assertEquals(0, v.numObjectElements());
});
}
@Test
public void testEmptyLargeObject() {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b1000010, 0x00, 0x00, 0x00, 0x00}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {0b1000010, 0x00, 0x00, 0x00, 0x00}), VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
Assert.assertEquals(0, v.numObjectElements());
});
}
@@ -229,14 +161,14 @@ public class TestVariantObject {
Variant value = new Variant(
ByteBuffer.wrap(constructObject(keys, fields, true)),
constructMetadata(false, ImmutableList.of("c", "b", "a")));
- testVariant(value, v -> {
- checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
Assert.assertEquals(3, v.numObjectElements());
- checkType(v.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ VariantTestUtil.checkType(v.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(1234567890, v.getFieldByKey("a").getInt());
- checkType(v.getFieldByKey("b"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ VariantTestUtil.checkType(v.getFieldByKey("b"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
Assert.assertTrue(v.getFieldByKey("b").getBoolean());
- checkType(v.getFieldByKey("c"), VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ VariantTestUtil.checkType(v.getFieldByKey("c"), VariantUtil.PRIMITIVE, Variant.Type.STRING);
Assert.assertEquals("variant", v.getFieldByKey("c").getString());
});
}
@@ -250,22 +182,22 @@ public class TestVariantObject {
Variant value = new Variant(
ByteBuffer.wrap(constructObject(keys, fields, true)),
constructMetadata(true, ImmutableList.of("a", "b", "c")));
- testVariant(value, v -> {
- checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
Assert.assertEquals(3, v.numObjectElements());
- checkType(v.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ VariantTestUtil.checkType(v.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(1234567890, v.getFieldByKey("a").getInt());
- checkType(v.getFieldByKey("b"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ VariantTestUtil.checkType(v.getFieldByKey("b"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
Assert.assertTrue(v.getFieldByKey("b").getBoolean());
- checkType(v.getFieldByKey("c"), VariantUtil.OBJECT, Variant.Type.OBJECT);
+ VariantTestUtil.checkType(v.getFieldByKey("c"), VariantUtil.OBJECT, Variant.Type.OBJECT);
Variant nestedV = v.getFieldByKey("c");
Assert.assertEquals(2, nestedV.numObjectElements());
- checkType(nestedV.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.DATE);
+ VariantTestUtil.checkType(nestedV.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.DATE);
Assert.assertEquals(
LocalDate.parse("2025-04-17"),
LocalDate.ofEpochDay(nestedV.getFieldByKey("a").getInt()));
- checkType(nestedV.getFieldByKey("c"), VariantUtil.PRIMITIVE, Variant.Type.NULL);
+ VariantTestUtil.checkType(nestedV.getFieldByKey("c"), VariantUtil.PRIMITIVE, Variant.Type.NULL);
});
}
@@ -277,14 +209,14 @@ public class TestVariantObject {
Variant value = new Variant(
ByteBuffer.wrap(constructObject(keys, fields, false)),
constructMetadata(true, ImmutableList.of("a", "b", "c")));
- testVariant(value, v -> {
- checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
Assert.assertEquals(3, v.numObjectElements());
- checkType(v.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ VariantTestUtil.checkType(v.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(1234567890, v.getFieldByKey("a").getInt());
- checkType(v.getFieldByKey("b"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ VariantTestUtil.checkType(v.getFieldByKey("b"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
Assert.assertTrue(v.getFieldByKey("b").getBoolean());
- checkType(v.getFieldByKey("c"), VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ VariantTestUtil.checkType(v.getFieldByKey("c"), VariantUtil.PRIMITIVE, Variant.Type.STRING);
Assert.assertEquals("variant", v.getFieldByKey("c").getString());
});
}
@@ -293,17 +225,18 @@ public class TestVariantObject {
Variant value = new Variant(
ByteBuffer.wrap(constructObject(
ImmutableMap.of("a", 0, "b", 1, "c", 2),
- ImmutableMap.of("a", constructString(randomString), "b", VALUE_BOOL, "c", VALUE_INT),
+ ImmutableMap.of(
+ "a", VariantTestUtil.constructString(randomString), "b", VALUE_BOOL, "c", VALUE_INT),
true)),
constructMetadata(true, ImmutableList.of("a", "b", "c")));
- testVariant(value, v -> {
- checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
Assert.assertEquals(3, v.numObjectElements());
- checkType(v.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ VariantTestUtil.checkType(v.getFieldByKey("a"), VariantUtil.PRIMITIVE, Variant.Type.STRING);
Assert.assertEquals(randomString, v.getFieldByKey("a").getString());
- checkType(v.getFieldByKey("b"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ VariantTestUtil.checkType(v.getFieldByKey("b"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
Assert.assertTrue(v.getFieldByKey("b").getBoolean());
- checkType(v.getFieldByKey("c"), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ VariantTestUtil.checkType(v.getFieldByKey("c"), VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(1234567890, v.getFieldByKey("c").getInt());
});
}
@@ -311,19 +244,19 @@ public class TestVariantObject {
@Test
public void testObjectTwoByteOffset() {
// a string larger than 255 bytes to push the offset size above 1 byte
- testObjectOffsetSize(randomString(300));
+ testObjectOffsetSize(VariantTestUtil.randomString(300));
}
@Test
public void testObjectThreeByteOffset() {
// a string larger than 65535 bytes to push the offset size above 2 bytes
- testObjectOffsetSize(randomString(70_000));
+ testObjectOffsetSize(VariantTestUtil.randomString(70_000));
}
@Test
public void testObjectFourByteOffset() {
// a string larger than 16777215 bytes to push the offset size above 3 bytes
- testObjectOffsetSize(randomString(16_800_000));
+ testObjectOffsetSize(VariantTestUtil.randomString(16_800_000));
}
private void testObjectFieldIdSize(int numExtraKeys) {
@@ -340,12 +273,12 @@ public class TestVariantObject {
ImmutableMap.of("z1", VALUE_BOOL, "z2", VALUE_INT),
true)),
constructMetadata(true, fieldNames));
- testVariant(value, v -> {
- checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
Assert.assertEquals(2, v.numObjectElements());
- checkType(v.getFieldByKey("z1"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ VariantTestUtil.checkType(v.getFieldByKey("z1"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
Assert.assertTrue(v.getFieldByKey("z1").getBoolean());
- checkType(v.getFieldByKey("z2"), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ VariantTestUtil.checkType(v.getFieldByKey("z2"), VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(1234567890, v.getFieldByKey("z2").getInt());
});
}
@@ -375,7 +308,7 @@ public class TestVariantObject {
for (int i = 0; i < 1000; i++) {
String name = String.format("a%04d", i);
keys.put(name, i);
- fields.put(name, constructString(randomString(5)));
+ fields.put(name, VariantTestUtil.constructString(VariantTestUtil.randomString(5)));
}
List<String> sortedKeys = new ArrayList<>(keys.keySet());
@@ -383,13 +316,13 @@ public class TestVariantObject {
Variant value =
new Variant(ByteBuffer.wrap(constructObject(keys, fields, false)), constructMetadata(true, sortedKeys));
- testVariant(value, v -> {
- checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
Assert.assertEquals(1000, v.numObjectElements());
for (int i = 0; i < 1000; i++) {
String name = String.format("a%04d", i);
- checkType(v.getFieldByKey(name), VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ VariantTestUtil.checkType(v.getFieldByKey(name), VariantUtil.PRIMITIVE, Variant.Type.STRING);
Assert.assertEquals(
new String(fields.get(name), 5, fields.get(name).length - 5),
v.getFieldByKey(name).getString());
@@ -401,7 +334,7 @@ public class TestVariantObject {
public void testInvalidObject() {
try {
// An array header
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b10011}), EMPTY_METADATA);
+ Variant value = new Variant(ByteBuffer.wrap(new byte[] {0b10011}), VariantTestUtil.EMPTY_METADATA);
value.numObjectElements();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
diff --git a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java
new file mode 100644
index 000000000..9ca3d1a28
--- /dev/null
+++ b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantObjectBuilder.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestVariantObjectBuilder {
+ private static final Logger LOG = LoggerFactory.getLogger(TestVariantObjectBuilder.class);
+
+ @Test
+ public void testEmptyObjectBuilder() {
+ VariantBuilder b = new VariantBuilder();
+ b.startObject();
+ b.endObject();
+ VariantTestUtil.testVariant(b.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Assert.assertEquals(0, v.numObjectElements());
+ });
+ }
+
+ @Test
+ public void testLargeObjectBuilder() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder o = b.startObject();
+ for (int i = 0; i < 1234; i++) {
+ o.appendKey("a" + i);
+ o.appendLong(i);
+ }
+ b.endObject();
+ VariantTestUtil.testVariant(b.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Assert.assertEquals(1234, v.numObjectElements());
+ for (int i = 0; i < 1234; i++) {
+ VariantTestUtil.checkType(v.getFieldByKey("a" + i), VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ Assert.assertEquals(i, v.getFieldByKey("a" + i).getLong());
+ }
+ });
+ }
+
+ @Test
+ public void testMixedObjectBuilder() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder objBuilder = b.startObject();
+ objBuilder.appendKey("outer 2");
+ objBuilder.appendLong(1234567890);
+ objBuilder.appendKey("outer 1");
+ objBuilder.appendBoolean(true);
+ objBuilder.appendKey("outer 4");
+ objBuilder.startArray();
+ objBuilder.endArray();
+ objBuilder.appendKey("outer 3");
+ {
+ // build a nested obj
+ VariantObjectBuilder nestedBuilder = objBuilder.startObject();
+ nestedBuilder.appendKey("nested 3");
+ VariantArrayBuilder arr = nestedBuilder.startArray();
+ arr.appendInt(321);
+ nestedBuilder.endArray();
+ nestedBuilder.appendKey("nested 1");
+ {
+ // build a nested empty obj
+ nestedBuilder.startObject();
+ nestedBuilder.endObject();
+ }
+ nestedBuilder.appendKey("nested 2");
+ nestedBuilder.appendString("variant");
+ objBuilder.endObject();
+ }
+ b.endObject();
+
+ VariantTestUtil.testVariant(b.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Assert.assertEquals(4, v.numObjectElements());
+ VariantTestUtil.checkType(v.getFieldByKey("outer 1"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ Assert.assertTrue(v.getFieldByKey("outer 1").getBoolean());
+ VariantTestUtil.checkType(v.getFieldByKey("outer 2"), VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ Assert.assertEquals(1234567890, v.getFieldByKey("outer 2").getLong());
+ VariantTestUtil.checkType(v.getFieldByKey("outer 3"), VariantUtil.OBJECT, Variant.Type.OBJECT);
+
+ Variant nested = v.getFieldByKey("outer 3");
+ Assert.assertEquals(3, nested.numObjectElements());
+ VariantTestUtil.checkType(nested.getFieldByKey("nested 1"), VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Assert.assertEquals(0, nested.getFieldByKey("nested 1").numObjectElements());
+ VariantTestUtil.checkType(nested.getFieldByKey("nested 2"), VariantUtil.SHORT_STR, Variant.Type.STRING);
+ Assert.assertEquals("variant", nested.getFieldByKey("nested 2").getString());
+ VariantTestUtil.checkType(nested.getFieldByKey("nested 3"), VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Assert.assertEquals(1, nested.getFieldByKey("nested 3").numArrayElements());
+ VariantTestUtil.checkType(
+ nested.getFieldByKey("nested 3").getElementAtIndex(0), VariantUtil.PRIMITIVE, Variant.Type.INT);
+ Assert.assertEquals(
+ 321, nested.getFieldByKey("nested 3").getElementAtIndex(0).getInt());
+
+ VariantTestUtil.checkType(v.getFieldByKey("outer 4"), VariantUtil.ARRAY, Variant.Type.ARRAY);
+ Assert.assertEquals(0, v.getFieldByKey("outer 4").numArrayElements());
+ });
+ }
+
+ private void buildNested(int i, VariantObjectBuilder obj) {
+ if (i > 0) {
+ obj.appendKey("key" + i);
+ obj.appendString("str" + i);
+ obj.appendKey("duplicate");
+ buildNested(i - 1, obj.startObject());
+ obj.endObject();
+ }
+ }
+
+ @Test
+ public void testNestedBuilder() {
+ VariantBuilder b = new VariantBuilder();
+ buildNested(1000, b.startObject());
+ b.endObject();
+
+ VariantTestUtil.testVariant(b.build(), v -> {
+ Variant curr = v;
+ for (int i = 1000; i >= 0; i--) {
+ VariantTestUtil.checkType(curr, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ if (i == 0) {
+ Assert.assertEquals(0, curr.numObjectElements());
+ } else {
+ Assert.assertEquals(2, curr.numObjectElements());
+ VariantTestUtil.checkType(
+ curr.getFieldByKey("key" + i), VariantUtil.SHORT_STR, Variant.Type.STRING);
+ Assert.assertEquals("str" + i, curr.getFieldByKey("key" + i).getString());
+ curr = curr.getFieldByKey("duplicate");
+ }
+ }
+ });
+ }
+
+ private void testObjectOffsetSizeBuilder(String randomString) {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder objBuilder = b.startObject();
+ objBuilder.appendKey("key1");
+ objBuilder.appendString(randomString);
+ objBuilder.appendKey("key2");
+ objBuilder.appendBoolean(true);
+ objBuilder.appendKey("key3");
+ objBuilder.appendLong(1234567890);
+ b.endObject();
+
+ Variant v = b.build();
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Assert.assertEquals(3, v.numObjectElements());
+ VariantTestUtil.checkType(v.getFieldByKey("key1"), VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ Assert.assertEquals(randomString, v.getFieldByKey("key1").getString());
+ VariantTestUtil.checkType(v.getFieldByKey("key2"), VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ Assert.assertTrue(v.getFieldByKey("key2").getBoolean());
+ VariantTestUtil.checkType(v.getFieldByKey("key3"), VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ Assert.assertEquals(1234567890, v.getFieldByKey("key3").getLong());
+ }
+
+ @Test
+ public void testObjectTwoByteOffsetBuilder() {
+ // a string larger than 255 bytes to push the offset size above 1 byte
+ testObjectOffsetSizeBuilder(VariantTestUtil.randomString(300));
+ }
+
+ @Test
+ public void testObjectThreeByteOffsetBuilder() {
+ // a string larger than 65535 bytes to push the offset size above 2 bytes
+ testObjectOffsetSizeBuilder(VariantTestUtil.randomString(70_000));
+ }
+
+ @Test
+ public void testObjectFourByteOffsetBuilder() {
+ // a string larger than 16777215 bytes to push the offset size above 3 bytes
+ testObjectOffsetSizeBuilder(VariantTestUtil.randomString(16_800_000));
+ }
+
+ private void testObjectFieldIdSizeBuilder(int numKeys) {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder objBuilder = b.startObject();
+ for (int i = 0; i < numKeys; i++) {
+ objBuilder.appendKey("k" + i);
+ objBuilder.appendLong(i);
+ }
+ b.endObject();
+
+ Variant v = b.build();
+ VariantTestUtil.checkType(v, VariantUtil.OBJECT, Variant.Type.OBJECT);
+ Assert.assertEquals(numKeys, v.numObjectElements());
+ // Only check a few keys, to avoid slowing down the test
+ VariantTestUtil.checkType(v.getFieldByKey("k" + 0), VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ Assert.assertEquals(0, v.getFieldByKey("k" + 0).getLong());
+ VariantTestUtil.checkType(v.getFieldByKey("k" + (numKeys - 1)), VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ Assert.assertEquals(numKeys - 1, v.getFieldByKey("k" + (numKeys - 1)).getLong());
+ }
+
+ @Test
+ public void testObjectTwoByteFieldIdBuilder() {
+ // need more than 255 dictionary entries to push field id size above 1 byte
+ testObjectFieldIdSizeBuilder(300);
+ }
+
+ @Test
+ public void testObjectThreeByteFieldIdBuilder() {
+ // need more than 65535 dictionary entries to push field id size above 2 bytes
+ testObjectFieldIdSizeBuilder(70_000);
+ }
+
+ @Test
+ @Ignore("Test uses too much memory")
+ public void testObjectFourByteFieldIdBuilder() {
+ // need more than 16777215 dictionary entries to push field id size above 3 bytes
+ testObjectFieldIdSizeBuilder(16_800_000);
+ }
+
+ @Test
+ public void testDuplicateKeys() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder objBuilder = b.startObject();
+ objBuilder.appendKey("duplicate");
+ objBuilder.appendLong(0);
+ objBuilder.appendKey("duplicate");
+ objBuilder.appendLong(1);
+ b.endObject();
+ Variant v = b.build();
+ Assert.assertEquals(1, v.numObjectElements());
+ VariantTestUtil.checkType(v.getFieldByKey("duplicate"), VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ Assert.assertEquals(1, v.getFieldByKey("duplicate").getLong());
+ }
+
+ @Test
+ public void testSortingKeys() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder objBuilder = b.startObject();
+ objBuilder.appendKey("1");
+ objBuilder.appendString("1");
+ objBuilder.appendKey("0");
+ objBuilder.appendString("");
+ objBuilder.appendKey("3");
+ objBuilder.appendString("333");
+ objBuilder.appendKey("2");
+ objBuilder.appendString("22");
+ b.endObject();
+ Variant v = b.build();
+ Assert.assertEquals(4, v.numObjectElements());
+ VariantTestUtil.checkType(v.getFieldByKey("0"), VariantUtil.SHORT_STR, Variant.Type.STRING);
+ Assert.assertEquals("", v.getFieldByKey("0").getString());
+ VariantTestUtil.checkType(v.getFieldByKey("1"), VariantUtil.SHORT_STR, Variant.Type.STRING);
+ Assert.assertEquals("1", v.getFieldByKey("1").getString());
+ VariantTestUtil.checkType(v.getFieldByKey("2"), VariantUtil.SHORT_STR, Variant.Type.STRING);
+ Assert.assertEquals("22", v.getFieldByKey("2").getString());
+ VariantTestUtil.checkType(v.getFieldByKey("3"), VariantUtil.SHORT_STR, Variant.Type.STRING);
+ Assert.assertEquals("333", v.getFieldByKey("3").getString());
+ }
+
+ @Test
+ public void testMissingEndObject() {
+ VariantBuilder b = new VariantBuilder();
+ b.startObject();
+ try {
+ b.build();
+ Assert.fail("Expected Exception when calling build() without endObject()");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testMissingStartObject() {
+ VariantBuilder b = new VariantBuilder();
+ try {
+ b.endObject();
+ Assert.fail("Expected Exception when calling endObject() without startObject()");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testMissingValue() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ obj.appendKey("a");
+ try {
+ b.endObject();
+ Assert.fail("Expected Exception when calling endObject() with mismatched keys and values");
+ } catch (Exception e) {
+ // expected
+ }
+
+ obj.appendInt(1);
+ obj.appendKey("b");
+ try {
+ b.endObject();
+ Assert.fail("Expected Exception when calling endObject() with mismatched keys and values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testInvalidAppendDuringObjectAppend() {
+ VariantBuilder b = new VariantBuilder();
+ b.startObject();
+ try {
+ b.appendInt(1);
+ Assert.fail("Expected Exception when calling append() before endObject()");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testMultipleAppendKey() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ obj.appendKey("a");
+ try {
+ obj.appendKey("a");
+ Assert.fail("Expected Exception when calling appendKey() multiple times without appending a value");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testNoAppendKey() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ try {
+ obj.appendInt(1);
+ Assert.fail("Expected Exception when appending a value, before appending a key");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testMultipleAppendValue() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ obj.appendKey("a");
+ obj.appendInt(1);
+ try {
+ obj.appendInt(1);
+ Assert.fail("Expected Exception when appending a value, before appending a key");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testStartObjectEndArray() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ try {
+ obj.endArray();
+ Assert.fail("Expected Exception when calling endArray() while building object");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedObject() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ obj.appendKey("outer");
+ obj.startObject();
+ try {
+ b.endObject();
+ Assert.fail("Expected Exception when calling endObject() with an open nested object");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedObjectWithKey() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ obj.appendKey("outer");
+ VariantObjectBuilder nested = obj.startObject();
+ nested.appendKey("nested");
+ try {
+ b.endObject();
+ Assert.fail("Expected Exception when calling endObject() with an open nested object");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedObjectWithKeyValue() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ obj.appendKey("outer");
+ VariantObjectBuilder nested = obj.startObject();
+ nested.appendKey("nested");
+ nested.appendInt(1);
+ try {
+ b.endObject();
+ Assert.fail("Expected Exception when calling endObject() with an open nested object");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedArray() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ obj.appendKey("outer");
+ obj.startArray();
+ try {
+ b.endObject();
+ Assert.fail("Expected Exception when calling endObject() with an open nested array");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOpenNestedArrayWithElement() {
+ VariantBuilder b = new VariantBuilder();
+ VariantObjectBuilder obj = b.startObject();
+ obj.appendKey("outer");
+ VariantArrayBuilder nested = obj.startArray();
+ nested.appendInt(1);
+ try {
+ b.endObject();
+ Assert.fail("Expected Exception when calling endObject() with an open nested array");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+}
diff --git a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantScalar.java b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantScalar.java
index af3feeae8..6219d7d08 100644
--- a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantScalar.java
+++ b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantScalar.java
@@ -24,8 +24,7 @@ import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.temporal.ChronoUnit;
-import java.util.*;
-import java.util.function.Consumer;
+import java.util.UUID;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -34,78 +33,29 @@ import org.slf4j.LoggerFactory;
public class TestVariantScalar {
private static final Logger LOG = LoggerFactory.getLogger(TestVariantScalar.class);
- private static final ByteBuffer EMPTY_METADATA = ByteBuffer.wrap(new byte[] {0b1});
-
- private void checkType(Variant v, int expectedBasicType, Variant.Type expectedType) {
- Assert.assertEquals(expectedBasicType, v.value.get(v.value.position()) & VariantUtil.BASIC_TYPE_MASK);
- Assert.assertEquals(expectedType, v.getType());
- }
-
- private void testVariant(Variant v, Consumer<Variant> consumer) {
- consumer.accept(v);
- // Create new Variant with different byte offsets
- byte[] newValue = new byte[v.value.capacity() + 50];
- byte[] newMetadata = new byte[v.metadata.capacity() + 50];
- Arrays.fill(newValue, (byte) 0xFF);
- Arrays.fill(newMetadata, (byte) 0xFF);
- v.value.position(0);
- v.value.get(newValue, 25, v.value.capacity());
- v.value.position(0);
- v.metadata.position(0);
- v.metadata.get(newMetadata, 25, v.metadata.capacity());
- v.metadata.position(0);
- Variant v2 = new Variant(
- ByteBuffer.wrap(newValue, 25, v.value.capacity()),
- ByteBuffer.wrap(newMetadata, 25, v.metadata.capacity()));
- consumer.accept(v2);
- }
-
- private static byte primitiveHeader(int type) {
- return (byte) (type << 2);
- }
-
- private static byte metadataHeader(boolean isSorted, int offsetSize) {
- return (byte) (((offsetSize - 1) << 6) | (isSorted ? 0b10000 : 0) | 0b0001);
- }
-
- private static int getMinIntegerSize(int value) {
- return (value <= 0xFF) ? 1 : (value <= 0xFFFF) ? 2 : (value <= 0xFFFFFF) ? 3 : 4;
- }
-
- private static void writeVarlenInt(ByteBuffer buffer, int value, int valueSize) {
- if (valueSize == 1) {
- buffer.put((byte) value);
- } else if (valueSize == 2) {
- buffer.putShort((short) value);
- } else if (valueSize == 3) {
- buffer.put((byte) (value & 0xFF));
- buffer.put((byte) ((value >> 8) & 0xFF));
- buffer.put((byte) ((value >> 16) & 0xFF));
- } else {
- buffer.putInt(value);
- }
- }
-
@Test
public void testNull() {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(0)}), EMPTY_METADATA);
- testVariant(value, v -> checkType(v, VariantUtil.NULL, Variant.Type.NULL));
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(0)}), VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> VariantTestUtil.checkType(v, VariantUtil.NULL, Variant.Type.NULL));
}
@Test
public void testTrue() {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(1)}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(1)}), VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
Assert.assertTrue(v.getBoolean());
});
}
@Test
public void testFalse() {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(2)}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(2)}), VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
Assert.assertFalse(v.getBoolean());
});
}
@@ -114,11 +64,19 @@ public class TestVariantScalar {
public void testLong() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(6), (byte) 0xB1, 0x1C, 0x6C, (byte) 0xB1, (byte) 0xF4, 0x10, 0x22, 0x11
+ VariantTestUtil.primitiveHeader(6),
+ (byte) 0xB1,
+ 0x1C,
+ 0x6C,
+ (byte) 0xB1,
+ (byte) 0xF4,
+ 0x10,
+ 0x22,
+ 0x11
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.LONG);
Assert.assertEquals(1234567890987654321L, v.getLong());
});
}
@@ -127,7 +85,7 @@ public class TestVariantScalar {
public void testNegativeLong() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(6),
+ VariantTestUtil.primitiveHeader(6),
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
@@ -137,9 +95,9 @@ public class TestVariantScalar {
(byte) 0xFF,
(byte) 0xFF
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.LONG);
Assert.assertEquals(-1L, v.getLong());
});
}
@@ -147,9 +105,10 @@ public class TestVariantScalar {
@Test
public void testInt() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {primitiveHeader(5), (byte) 0xD2, 0x02, (byte) 0x96, 0x49}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.INT);
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(5), (byte) 0xD2, 0x02, (byte) 0x96, 0x49}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(1234567890, v.getInt());
});
}
@@ -157,48 +116,55 @@ public class TestVariantScalar {
@Test
public void testNegativeInt() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {primitiveHeader(5), (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.INT);
+ ByteBuffer.wrap(new byte[] {
+ VariantTestUtil.primitiveHeader(5), (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF
+ }),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.INT);
Assert.assertEquals(-1, v.getInt());
});
}
@Test
public void testShort() {
- Variant value =
- new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(4), (byte) 0xD2, 0x04}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.SHORT);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(4), (byte) 0xD2, 0x04}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.SHORT);
Assert.assertEquals((short) 1234, v.getShort());
});
}
@Test
public void testNegativeShort() {
- Variant value =
- new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(4), (byte) 0xFF, (byte) 0xFF}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.SHORT);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(4), (byte) 0xFF, (byte) 0xFF}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.SHORT);
Assert.assertEquals((short) -1, v.getShort());
});
}
@Test
public void testByte() {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(3), 34}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BYTE);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(3), 34}), VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BYTE);
Assert.assertEquals((byte) 34, v.getByte());
});
}
@Test
public void testNegativeByte() {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(3), (byte) 0xFF}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BYTE);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(3), (byte) 0xFF}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BYTE);
Assert.assertEquals((byte) -1, v.getByte());
});
}
@@ -206,10 +172,10 @@ public class TestVariantScalar {
@Test
public void testFloat() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {primitiveHeader(14), (byte) 0xD2, 0x02, (byte) 0x96, 0x49}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.FLOAT);
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(14), (byte) 0xD2, 0x02, (byte) 0x96, 0x49}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.FLOAT);
Assert.assertEquals(Float.intBitsToFloat(1234567890), v.getFloat(), 0);
});
}
@@ -217,9 +183,10 @@ public class TestVariantScalar {
@Test
public void testNegativeFloat() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {primitiveHeader(14), 0x00, 0x00, 0x00, (byte) 0x80}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.FLOAT);
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(14), 0x00, 0x00, 0x00, (byte) 0x80}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.FLOAT);
Assert.assertEquals(-0.0F, v.getFloat(), 0);
});
}
@@ -228,11 +195,19 @@ public class TestVariantScalar {
public void testDouble() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(7), (byte) 0xB1, 0x1C, 0x6C, (byte) 0xB1, (byte) 0xF4, 0x10, 0x22, 0x11
+ VariantTestUtil.primitiveHeader(7),
+ (byte) 0xB1,
+ 0x1C,
+ 0x6C,
+ (byte) 0xB1,
+ (byte) 0xF4,
+ 0x10,
+ 0x22,
+ 0x11
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DOUBLE);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DOUBLE);
Assert.assertEquals(Double.longBitsToDouble(1234567890987654321L), v.getDouble(), 0);
});
}
@@ -240,10 +215,12 @@ public class TestVariantScalar {
@Test
public void testNegativeDouble() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {primitiveHeader(7), 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, (byte) 0x80}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DOUBLE);
+ ByteBuffer.wrap(new byte[] {
+ VariantTestUtil.primitiveHeader(7), 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, (byte) 0x80
+ }),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DOUBLE);
Assert.assertEquals(-0.0D, v.getDouble(), 0);
});
}
@@ -251,10 +228,11 @@ public class TestVariantScalar {
@Test
public void testDecimal4() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {primitiveHeader(8), 0x04, (byte) 0xD2, 0x02, (byte) 0x96, 0x49}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL4);
+ ByteBuffer.wrap(
+ new byte[] {VariantTestUtil.primitiveHeader(8), 0x04, (byte) 0xD2, 0x02, (byte) 0x96, 0x49}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL4);
Assert.assertEquals(new BigDecimal("123456.7890"), v.getDecimal());
});
}
@@ -262,11 +240,12 @@ public class TestVariantScalar {
@Test
public void testNegativeDecimal4() {
Variant value = new Variant(
- ByteBuffer.wrap(
- new byte[] {primitiveHeader(8), 0x04, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL4);
+ ByteBuffer.wrap(new byte[] {
+ VariantTestUtil.primitiveHeader(8), 0x04, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF
+ }),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL4);
Assert.assertEquals(new BigDecimal("-0.0001"), v.getDecimal());
});
}
@@ -275,11 +254,20 @@ public class TestVariantScalar {
public void testDecimal8() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(9), 0x09, (byte) 0xB1, 0x1C, 0x6C, (byte) 0xB1, (byte) 0xF4, 0x10, 0x22, 0x11
+ VariantTestUtil.primitiveHeader(9),
+ 0x09,
+ (byte) 0xB1,
+ 0x1C,
+ 0x6C,
+ (byte) 0xB1,
+ (byte) 0xF4,
+ 0x10,
+ 0x22,
+ 0x11
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL8);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL8);
Assert.assertEquals(new BigDecimal("1234567890.987654321"), v.getDecimal());
});
}
@@ -288,7 +276,7 @@ public class TestVariantScalar {
public void testNegativeDecimal8() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(9),
+ VariantTestUtil.primitiveHeader(9),
0x09,
(byte) 0xFF,
(byte) 0xFF,
@@ -299,9 +287,9 @@ public class TestVariantScalar {
(byte) 0xFF,
(byte) 0xFF
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL8);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL8);
Assert.assertEquals(new BigDecimal("-0.000000001"), v.getDecimal());
});
}
@@ -310,7 +298,7 @@ public class TestVariantScalar {
public void testDecimal16() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(10),
+ VariantTestUtil.primitiveHeader(10),
0x09,
0x15,
0x71,
@@ -329,9 +317,9 @@ public class TestVariantScalar {
0x00,
0x00
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL16);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL16);
Assert.assertEquals(new BigDecimal("9876543210.123456789"), v.getDecimal());
});
}
@@ -340,7 +328,7 @@ public class TestVariantScalar {
public void testNegativeDecimal16() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(10),
+ VariantTestUtil.primitiveHeader(10),
0x09,
(byte) 0xEB,
(byte) 0x8E,
@@ -359,9 +347,9 @@ public class TestVariantScalar {
(byte) 0xFF,
(byte) 0xFF
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL16);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL16);
Assert.assertEquals(new BigDecimal("-9876543210.123456789"), v.getDecimal());
});
}
@@ -369,9 +357,10 @@ public class TestVariantScalar {
@Test
public void testDate() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {primitiveHeader(11), (byte) 0xE3, 0x4E, 0x00, 0x00}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DATE);
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(11), (byte) 0xE3, 0x4E, 0x00, 0x00}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DATE);
Assert.assertEquals(LocalDate.parse("2025-04-17"), LocalDate.ofEpochDay(v.getInt()));
});
}
@@ -379,33 +368,43 @@ public class TestVariantScalar {
@Test
public void testNegativeDate() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {primitiveHeader(11), (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DATE);
+ ByteBuffer.wrap(new byte[] {
+ VariantTestUtil.primitiveHeader(11), (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF
+ }),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DATE);
Assert.assertEquals(LocalDate.parse("1969-12-31"), LocalDate.ofEpochDay(v.getInt()));
});
}
@Test
- public void testTimestamp() {
+ public void testTimestampTz() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(12), (byte) 0xC0, 0x77, (byte) 0xA1, (byte) 0xEA, (byte) 0xF4, 0x32, 0x06, 0x00
+ VariantTestUtil.primitiveHeader(12),
+ (byte) 0xC0,
+ 0x77,
+ (byte) 0xA1,
+ (byte) 0xEA,
+ (byte) 0xF4,
+ 0x32,
+ 0x06,
+ 0x00
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_TZ);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_TZ);
Assert.assertEquals(
Instant.parse("2025-04-17T08:09:10.123456Z"), Instant.EPOCH.plus(v.getLong(), ChronoUnit.MICROS));
});
}
@Test
- public void testNegativeTimestamp() {
+ public void testNegativeTimestampTz() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(12),
+ VariantTestUtil.primitiveHeader(12),
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
@@ -415,9 +414,9 @@ public class TestVariantScalar {
(byte) 0xFF,
(byte) 0xFF
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_TZ);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_TZ);
Assert.assertEquals(
Instant.parse("1969-12-31T23:59:59.999999Z"), Instant.EPOCH.plus(v.getLong(), ChronoUnit.MICROS));
});
@@ -427,11 +426,19 @@ public class TestVariantScalar {
public void testTimestampNtz() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(13), (byte) 0xC0, 0x77, (byte) 0xA1, (byte) 0xEA, (byte) 0xF4, 0x32, 0x06, 0x00
+ VariantTestUtil.primitiveHeader(13),
+ (byte) 0xC0,
+ 0x77,
+ (byte) 0xA1,
+ (byte) 0xEA,
+ (byte) 0xF4,
+ 0x32,
+ 0x06,
+ 0x00
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NTZ);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NTZ);
Assert.assertEquals(
Instant.parse("2025-04-17T08:09:10.123456Z"), Instant.EPOCH.plus(v.getLong(), ChronoUnit.MICROS));
});
@@ -441,7 +448,7 @@ public class TestVariantScalar {
public void testNegativeTimestampNtz() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(13),
+ VariantTestUtil.primitiveHeader(13),
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
@@ -451,9 +458,9 @@ public class TestVariantScalar {
(byte) 0xFF,
(byte) 0xFF
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NTZ);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NTZ);
Assert.assertEquals(
Instant.parse("1969-12-31T23:59:59.999999Z"), Instant.EPOCH.plus(v.getLong(), ChronoUnit.MICROS));
});
@@ -462,10 +469,12 @@ public class TestVariantScalar {
@Test
public void testBinary() {
Variant value = new Variant(
- ByteBuffer.wrap(new byte[] {primitiveHeader(15), 0x05, 0x00, 0x00, 0x00, 'a', 'b', 'c', 'd', 'e'}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BINARY);
+ ByteBuffer.wrap(
+ new byte[] {VariantTestUtil.primitiveHeader(15), 0x05, 0x00, 0x00, 0x00, 'a', 'b', 'c', 'd', 'e'
+ }),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BINARY);
Assert.assertEquals(ByteBuffer.wrap(new byte[] {'a', 'b', 'c', 'd', 'e'}), v.getBinary());
});
}
@@ -473,21 +482,23 @@ public class TestVariantScalar {
@Test
public void testString() {
Variant value = new Variant(
- ByteBuffer.wrap(
- new byte[] {primitiveHeader(16), 0x07, 0x00, 0x00, 0x00, 'v', 'a', 'r', 'i', 'a', 'n', 't'}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ ByteBuffer.wrap(new byte[] {
+ VariantTestUtil.primitiveHeader(16), 0x07, 0x00, 0x00, 0x00, 'v', 'a', 'r', 'i', 'a', 'n', 't'
+ }),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.STRING);
Assert.assertEquals("variant", v.getString());
});
}
@Test
public void testShortString() {
- Variant value =
- new Variant(ByteBuffer.wrap(new byte[] {0b11101, 'v', 'a', 'r', 'i', 'a', 'n', 't'}), EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.SHORT_STR, Variant.Type.STRING);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {0b11101, 'v', 'a', 'r', 'i', 'a', 'n', 't'}),
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.SHORT_STR, Variant.Type.STRING);
Assert.assertEquals("variant", v.getString());
});
}
@@ -496,7 +507,7 @@ public class TestVariantScalar {
public void testTime() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(17),
+ VariantTestUtil.primitiveHeader(17),
(byte) 0x00,
(byte) 0x00,
(byte) 0xCA,
@@ -506,32 +517,40 @@ public class TestVariantScalar {
(byte) 0x00,
(byte) 0x00
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIME);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIME);
Assert.assertEquals(LocalTime.parse("23:59:59.123456"), LocalTime.ofNanoOfDay(v.getLong() * 1_000));
});
}
@Test
- public void testTimestampNanos() {
+ public void testTimestampNanosTz() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(18), 0x15, (byte) 0xC9, (byte) 0xBB, (byte) 0x86, (byte) 0xB4, 0x0C, 0x37, 0x18
+ VariantTestUtil.primitiveHeader(18),
+ 0x15,
+ (byte) 0xC9,
+ (byte) 0xBB,
+ (byte) 0x86,
+ (byte) 0xB4,
+ 0x0C,
+ 0x37,
+ 0x18
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_TZ);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_TZ);
Assert.assertEquals(
Instant.parse("2025-04-17T08:09:10.123456789Z"), Instant.EPOCH.plus(v.getLong(), ChronoUnit.NANOS));
});
}
@Test
- public void testNegativeTimestampNanos() {
+ public void testNegativeTimestampNanosTz() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(18),
+ VariantTestUtil.primitiveHeader(18),
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
@@ -541,9 +560,9 @@ public class TestVariantScalar {
(byte) 0xFF,
(byte) 0xFF
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_TZ);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_TZ);
Assert.assertEquals(
Instant.parse("1969-12-31T23:59:59.999999999Z"), Instant.EPOCH.plus(v.getLong(), ChronoUnit.NANOS));
});
@@ -553,11 +572,19 @@ public class TestVariantScalar {
public void testTimestampNanosNtz() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(19), 0x15, (byte) 0xC9, (byte) 0xBB, (byte) 0x86, (byte) 0xB4, 0x0C, 0x37, 0x18
+ VariantTestUtil.primitiveHeader(19),
+ 0x15,
+ (byte) 0xC9,
+ (byte) 0xBB,
+ (byte) 0x86,
+ (byte) 0xB4,
+ 0x0C,
+ 0x37,
+ 0x18
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_NTZ);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_NTZ);
Assert.assertEquals(
Instant.parse("2025-04-17T08:09:10.123456789Z"), Instant.EPOCH.plus(v.getLong(), ChronoUnit.NANOS));
});
@@ -567,7 +594,7 @@ public class TestVariantScalar {
public void testNegativeTimestampNanosNtz() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(19),
+ VariantTestUtil.primitiveHeader(19),
(byte) 0xFF,
(byte) 0xFF,
(byte) 0xFF,
@@ -577,9 +604,9 @@ public class TestVariantScalar {
(byte) 0xFF,
(byte) 0xFF
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_NTZ);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_NTZ);
Assert.assertEquals(
Instant.parse("1969-12-31T23:59:59.999999999Z"), Instant.EPOCH.plus(v.getLong(), ChronoUnit.NANOS));
});
@@ -589,7 +616,7 @@ public class TestVariantScalar {
public void testUUID() {
Variant value = new Variant(
ByteBuffer.wrap(new byte[] {
- primitiveHeader(20),
+ VariantTestUtil.primitiveHeader(20),
0x00,
0x11,
0x22,
@@ -607,9 +634,9 @@ public class TestVariantScalar {
(byte) 0xEE,
(byte) 0xFF
}),
- EMPTY_METADATA);
- testVariant(value, v -> {
- checkType(v, VariantUtil.PRIMITIVE, Variant.Type.UUID);
+ VariantTestUtil.EMPTY_METADATA);
+ VariantTestUtil.testVariant(value, v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.UUID);
Assert.assertEquals(UUID.fromString("00112233-4455-6677-8899-aabbccddeeff"), v.getUUID());
});
}
@@ -617,7 +644,7 @@ public class TestVariantScalar {
@Test
public void testInvalidType() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {(byte) 0xFC}), EMPTY_METADATA);
+ Variant value = new Variant(ByteBuffer.wrap(new byte[] {(byte) 0xFC}), VariantTestUtil.EMPTY_METADATA);
value.getBoolean();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -629,7 +656,8 @@ public class TestVariantScalar {
@Test
public void testInvalidBoolean() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6)}), VariantTestUtil.EMPTY_METADATA);
value.getBoolean();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -640,7 +668,8 @@ public class TestVariantScalar {
@Test
public void testInvalidLong() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(16)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(16)}), VariantTestUtil.EMPTY_METADATA);
value.getLong();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -653,7 +682,8 @@ public class TestVariantScalar {
@Test
public void testInvalidInt() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6)}), VariantTestUtil.EMPTY_METADATA);
value.getInt();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -664,7 +694,8 @@ public class TestVariantScalar {
@Test
public void testInvalidShort() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6)}), VariantTestUtil.EMPTY_METADATA);
value.getShort();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -675,7 +706,8 @@ public class TestVariantScalar {
@Test
public void testInvalidByte() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6)}), VariantTestUtil.EMPTY_METADATA);
value.getByte();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -686,7 +718,8 @@ public class TestVariantScalar {
@Test
public void testInvalidFloat() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6)}), VariantTestUtil.EMPTY_METADATA);
value.getFloat();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -697,7 +730,8 @@ public class TestVariantScalar {
@Test
public void testInvalidDouble() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6)}), VariantTestUtil.EMPTY_METADATA);
value.getDouble();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -708,7 +742,9 @@ public class TestVariantScalar {
@Test
public void testInvalidDecimal() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6), 0}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6), 0}),
+ VariantTestUtil.EMPTY_METADATA);
value.getDecimal();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -719,7 +755,8 @@ public class TestVariantScalar {
@Test
public void testInvalidUUID() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6)}), VariantTestUtil.EMPTY_METADATA);
value.getUUID();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -730,7 +767,8 @@ public class TestVariantScalar {
@Test
public void testInvalidString() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6)}), VariantTestUtil.EMPTY_METADATA);
value.getString();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
@@ -741,7 +779,8 @@ public class TestVariantScalar {
@Test
public void testInvalidBinary() {
try {
- Variant value = new Variant(ByteBuffer.wrap(new byte[] {primitiveHeader(6)}), EMPTY_METADATA);
+ Variant value = new Variant(
+ ByteBuffer.wrap(new byte[] {VariantTestUtil.primitiveHeader(6)}), VariantTestUtil.EMPTY_METADATA);
value.getBinary();
Assert.fail("Expected exception not thrown");
} catch (Exception e) {
diff --git a/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantScalarBuilder.java b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantScalarBuilder.java
new file mode 100644
index 000000000..e6e3d322e
--- /dev/null
+++ b/parquet-variant/src/test/java/org/apache/parquet/variant/TestVariantScalarBuilder.java
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.util.Arrays;
+import java.util.UUID;
+import java.util.stream.IntStream;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestVariantScalarBuilder {
+ private static final Logger LOG = LoggerFactory.getLogger(TestVariantScalarBuilder.class);
+
+ @Test
+ public void testNullBuilder() {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendNull();
+ VariantTestUtil.testVariant(vb.build(), v -> VariantTestUtil.checkType(v, VariantUtil.NULL, Variant.Type.NULL));
+
+ try {
+ vb.appendNull();
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testBooleanBuilder() {
+ Arrays.asList(true, false).forEach(b -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendBoolean(b);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BOOLEAN);
+ Assert.assertEquals(b, v.getBoolean());
+ });
+
+ try {
+ vb.appendBoolean(true);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ });
+ }
+
+ @Test
+ public void testLongBuilder() {
+ Arrays.asList(
+ 0L,
+ (long) Byte.MIN_VALUE,
+ (long) Byte.MAX_VALUE,
+ (long) Short.MIN_VALUE,
+ (long) Short.MAX_VALUE,
+ (long) Integer.MIN_VALUE,
+ (long) Integer.MAX_VALUE,
+ Long.MIN_VALUE,
+ Long.MAX_VALUE)
+ .forEach(l -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendLong(l);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.LONG);
+ Assert.assertEquals((long) l, v.getLong());
+ });
+
+ try {
+ vb.appendLong(1L);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ });
+ }
+
+ @Test
+ public void testIntBuilder() {
+ Arrays.asList(
+ 0,
+ (int) Byte.MIN_VALUE,
+ (int) Byte.MAX_VALUE,
+ (int) Short.MIN_VALUE,
+ (int) Short.MAX_VALUE,
+ Integer.MIN_VALUE,
+ Integer.MAX_VALUE)
+ .forEach(i -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendInt(i);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.INT);
+ Assert.assertEquals((int) i, v.getInt());
+ });
+
+ try {
+ vb.appendInt(1);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ });
+ }
+
+ @Test
+ public void testShortBuilder() {
+ Arrays.asList((short) 0, (short) Byte.MIN_VALUE, (short) Byte.MAX_VALUE, Short.MIN_VALUE, Short.MAX_VALUE)
+ .forEach(s -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendShort(s);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.SHORT);
+ Assert.assertEquals((short) s, v.getShort());
+ });
+
+ try {
+ vb.appendShort((short) 1);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ });
+ }
+
+ @Test
+ public void testByteBuilder() {
+ Arrays.asList((byte) 0, Byte.MIN_VALUE, Byte.MAX_VALUE).forEach(b -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendByte(b);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BYTE);
+ Assert.assertEquals((byte) b, v.getByte());
+ });
+
+ try {
+ vb.appendByte((byte) 1);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ });
+ }
+
+ @Test
+ public void testFloatBuilder() {
+ Arrays.asList(Float.MIN_VALUE, 0f, -0f, Float.MAX_VALUE).forEach(f -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendFloat(f);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.FLOAT);
+ Assert.assertEquals(f, v.getFloat(), 0);
+ });
+
+ try {
+ vb.appendFloat(1.2f);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ });
+ }
+
+ @Test
+ public void testDoubleBuilder() {
+ Arrays.asList(Double.MIN_VALUE, 0d, -0d, Double.MAX_VALUE).forEach(d -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendDouble(d);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DOUBLE);
+ Assert.assertEquals(d, v.getDouble(), 0);
+ });
+
+ try {
+ vb.appendDouble(1.2);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ });
+ }
+
+ @Test
+ public void testDecimalBuilder() {
+ // decimal4
+ Arrays.asList(new BigDecimal("123.456"), new BigDecimal("-987.654")).forEach(d -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendDecimal(d);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL4);
+ Assert.assertEquals(d, v.getDecimal());
+ });
+ });
+
+ // decimal8
+ Arrays.asList(new BigDecimal("10.2147483647"), new BigDecimal("-1021474836.47"))
+ .forEach(d -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendDecimal(d);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL8);
+ Assert.assertEquals(d, v.getDecimal());
+ });
+ });
+
+ // decimal16
+ Arrays.asList(new BigDecimal("109223372036854775.807"), new BigDecimal("-109.223372036854775807"))
+ .forEach(d -> {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendDecimal(d);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DECIMAL16);
+ Assert.assertEquals(d, v.getDecimal());
+ });
+ });
+
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendDecimal(new BigDecimal("10.2147483647"));
+ try {
+ vb.appendDecimal(new BigDecimal("10.2147483647"));
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testDateBuilder() {
+ VariantBuilder vb = new VariantBuilder();
+ int days = Math.toIntExact(LocalDate.of(2024, 12, 16).toEpochDay());
+ vb.appendDate(days);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.DATE);
+ Assert.assertEquals(days, v.getInt());
+ });
+
+ try {
+ vb.appendDate(123);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testTimestampTzBuilder() {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendTimestampTz(1734373425321456L);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_TZ);
+ Assert.assertEquals(1734373425321456L, v.getLong());
+ });
+
+ try {
+ vb.appendTimestampTz(1734373425321456L);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testTimestampNtzBuilder() {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendTimestampNtz(1734373425321456L);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NTZ);
+ Assert.assertEquals(1734373425321456L, v.getLong());
+ });
+
+ try {
+ vb.appendTimestampNtz(1734373425321456L);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testBinaryBuilder() {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendBinary(ByteBuffer.wrap(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}));
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.BINARY);
+ Assert.assertEquals(ByteBuffer.wrap(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}), v.getBinary());
+ });
+
+ try {
+ vb.appendBinary(ByteBuffer.wrap(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}));
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testStringBuilder() {
+ IntStream.range(VariantUtil.MAX_SHORT_STR_SIZE - 3, VariantUtil.MAX_SHORT_STR_SIZE + 3)
+ .forEach(len -> {
+ VariantBuilder vb = new VariantBuilder();
+ String s = VariantTestUtil.randomString(len);
+ vb.appendString(s);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ if (len <= VariantUtil.MAX_SHORT_STR_SIZE) {
+ VariantTestUtil.checkType(v, VariantUtil.SHORT_STR, Variant.Type.STRING);
+ } else {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.STRING);
+ }
+ Assert.assertEquals(s, v.getString());
+ });
+ });
+
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendString(VariantTestUtil.randomString(10));
+ try {
+ vb.appendString(VariantTestUtil.randomString(10));
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testTimeBuilder() {
+ for (String timeStr : Arrays.asList(
+ "00:00:00.000000", "00:00:00.000120", "12:00:00.000000", "12:00:00.002300", "23:59:59.999999")) {
+ VariantBuilder vb = new VariantBuilder();
+ long micros = LocalTime.parse(timeStr).toNanoOfDay() / 1_000;
+ vb.appendTime(micros);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIME);
+ Assert.assertEquals(micros, v.getLong());
+ });
+ }
+
+ // test negative time
+ try {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendTime(-1);
+ Assert.fail("Expected Exception when adding a negative time value");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendTime(123456);
+ try {
+ vb.appendTime(123456);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testTimestampNanosBuilder() {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendTimestampNanosTz(1734373425321456987L);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_TZ);
+ Assert.assertEquals(1734373425321456987L, v.getLong());
+ });
+
+ try {
+ vb.appendTimestampNanosTz(1734373425321456987L);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testTimestampNanosNtzBuilder() {
+ VariantBuilder vb = new VariantBuilder();
+ vb.appendTimestampNanosNtz(1734373425321456987L);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.TIMESTAMP_NANOS_NTZ);
+ Assert.assertEquals(1734373425321456987L, v.getLong());
+ });
+
+ try {
+ vb.appendTimestampNanosNtz(1734373425321456987L);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testUUIDBuilder() {
+ VariantBuilder vb = new VariantBuilder();
+ byte[] uuid = new byte[] {0, 17, 34, 51, 68, 85, 102, 119, -120, -103, -86, -69, -52, -35, -18, -1};
+ long msb = ByteBuffer.wrap(uuid, 0, 8).order(ByteOrder.BIG_ENDIAN).getLong();
+ long lsb = ByteBuffer.wrap(uuid, 8, 8).order(ByteOrder.BIG_ENDIAN).getLong();
+ UUID expected = new UUID(msb, lsb);
+
+ vb.appendUUID(expected);
+ VariantTestUtil.testVariant(vb.build(), v -> {
+ VariantTestUtil.checkType(v, VariantUtil.PRIMITIVE, Variant.Type.UUID);
+ Assert.assertEquals(expected, v.getUUID());
+ });
+
+ try {
+ vb.appendUUID(expected);
+ Assert.fail("Expected Exception when appending multiple values");
+ } catch (Exception e) {
+ // expected
+ }
+ }
+}
diff --git a/parquet-variant/src/test/java/org/apache/parquet/variant/VariantTestUtil.java b/parquet-variant/src/test/java/org/apache/parquet/variant/VariantTestUtil.java
new file mode 100644
index 000000000..a438714f0
--- /dev/null
+++ b/parquet-variant/src/test/java/org/apache/parquet/variant/VariantTestUtil.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.security.SecureRandom;
+import java.util.Arrays;
+import java.util.function.Consumer;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class VariantTestUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(VariantTestUtil.class);
+ private static final String RANDOM_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+
+ /** Random number generator for generating random strings */
+ private static SecureRandom random = new SecureRandom(new byte[] {1, 2, 3, 4, 5});
+
+ static final ByteBuffer EMPTY_METADATA = ByteBuffer.wrap(new byte[] {0b1});
+
+ static void checkType(Variant v, int expectedBasicType, Variant.Type expectedType) {
+ Assert.assertEquals(expectedBasicType, v.value.get(v.value.position()) & VariantUtil.BASIC_TYPE_MASK);
+ Assert.assertEquals(expectedType, v.getType());
+ }
+
+ static String randomString(int len) {
+ StringBuilder sb = new StringBuilder(len);
+ for (int i = 0; i < len; i++) {
+ sb.append(RANDOM_CHARS.charAt(random.nextInt(RANDOM_CHARS.length())));
+ }
+ return sb.toString();
+ }
+
+ static void testVariant(Variant v, Consumer<Variant> consumer) {
+ consumer.accept(v);
+ // Create new Variant with different byte offsets
+ byte[] newValue = new byte[v.value.capacity() + 50];
+ byte[] newMetadata = new byte[v.metadata.capacity() + 50];
+ Arrays.fill(newValue, (byte) 0xFF);
+ Arrays.fill(newMetadata, (byte) 0xFF);
+ v.value.position(0);
+ v.value.get(newValue, 25, v.value.capacity());
+ v.value.position(0);
+ v.metadata.position(0);
+ v.metadata.get(newMetadata, 25, v.metadata.capacity());
+ v.metadata.position(0);
+ Variant v2 = new Variant(
+ ByteBuffer.wrap(newValue, 25, v.value.capacity()),
+ ByteBuffer.wrap(newMetadata, 25, v.metadata.capacity()));
+ consumer.accept(v2);
+ }
+
+ static byte primitiveHeader(int type) {
+ return (byte) (type << 2);
+ }
+
+ static byte metadataHeader(boolean isSorted, int offsetSize) {
+ return (byte) (((offsetSize - 1) << 6) | (isSorted ? 0b10000 : 0) | 0b0001);
+ }
+
+ static byte[] constructString(String value) {
+ return ByteBuffer.allocate(value.length() + 5)
+ .order(ByteOrder.LITTLE_ENDIAN)
+ .put(primitiveHeader(16))
+ .putInt(value.length())
+ .put(value.getBytes(StandardCharsets.UTF_8))
+ .array();
+ }
+
+ static int getMinIntegerSize(int value) {
+ return (value <= 0xFF) ? 1 : (value <= 0xFFFF) ? 2 : (value <= 0xFFFFFF) ? 3 : 4;
+ }
+
+ static void writeVarlenInt(ByteBuffer buffer, int value, int valueSize) {
+ if (valueSize == 1) {
+ buffer.put((byte) value);
+ } else if (valueSize == 2) {
+ buffer.putShort((short) value);
+ } else if (valueSize == 3) {
+ buffer.put((byte) (value & 0xFF));
+ buffer.put((byte) ((value >> 8) & 0xFF));
+ buffer.put((byte) ((value >> 16) & 0xFF));
+ } else {
+ buffer.putInt(value);
+ }
+ }
+}