You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@parquet.apache.org by "gene-db (via GitHub)" <gi...@apache.org> on 2025/02/05 23:56:52 UTC
Re: [PR] GH-3116: Implement the Variant binary encoding [parquet-java]
gene-db commented on code in PR #3117:
URL: https://github.com/apache/parquet-java/pull/3117#discussion_r1939791666
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+import static java.time.temporal.ChronoField.*;
+import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
+import static org.apache.parquet.variant.VariantUtil.*;
Review Comment:
Removed the static imports.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+import static java.time.temporal.ChronoField.*;
+import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
Review Comment:
Done.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+import static java.time.temporal.ChronoField.*;
+import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos,
+ (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ // Use unsigned right shift to compute the middle of `low` and `high`. This is not only a
+ // performance optimization, because it can properly handle the case where `low + high`
+ // overflows int.
+ int mid = (low + high) >>> 1;
+ int id = readUnsigned(value, idStart + idSize * mid, idSize);
+ int cmp = getMetadataKey(metadata, id).compareTo(key);
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ int offset = readUnsigned(value, offsetStart + offsetSize * mid, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ }
+ return null;
+ });
+ }
+
+ /**
+ * A field in a Variant object.
+ */
+ public static final class ObjectField {
+ public final String key;
+ public final Variant value;
+
+ public ObjectField(String key, Variant value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Get the object field at the `index` slot. Return null if `index` is out of the bound of
+ // `[0, objectSize())`.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+ /**
+ * Returns the object field at the `index` slot. Return null if `index` is out of the bound of
+ * `[0, objectSize())`. `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get
+ * @return the Objectfield at the `index` slot, or null if `index` is out of bounds
+ */
+ public ObjectField getFieldAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int id = readUnsigned(value, idStart + idSize * index, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ String key = getMetadataKey(metadata, id);
+ Variant v = new Variant(value, metadata, dataStart + offset);
+ return new ObjectField(key, v);
+ });
+ }
+
+ /**
+ * Returns the dictionary ID for the object field at the `index` slot.
+ * `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get the dictionary ID for
+ * @return the dictionary ID for the object field at the `index` slot
+ * @throws MalformedVariantException if `index` is out of bounds
+ */
+ public int getDictionaryIdAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) {
+ throw malformedVariant();
+ }
Review Comment:
`getFieldAtIndex` is a little bit different, since if a field doesn't exist in a variant value, that doesn't mean the variant value is malformed. This dictionary case is different because we are expecting an id in the dictionary to exist, but it doesn't.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
Review Comment:
Added a test case for getting the object keys, for bigger and smaller objects.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ // Use unsigned right shift to compute the middle of `low` and `high`. This is not only a
+ // performance optimization, because it can properly handle the case where `low + high`
+ // overflows int.
+ int mid = (low + high) >>> 1;
+ int id = readUnsigned(value, idStart + idSize * mid, idSize);
+ int cmp = getMetadataKey(metadata, id).compareTo(key);
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ int offset = readUnsigned(value, offsetStart + offsetSize * mid, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ }
+ return null;
+ });
+ }
+
+ /**
+ * A field in a Variant object.
+ */
+ public static final class ObjectField {
+ public final String key;
+ public final Variant value;
+
+ public ObjectField(String key, Variant value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Get the object field at the `index` slot. Return null if `index` is out of the bound of
+ // `[0, objectSize())`.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+ /**
+ * Returns the object field at the `index` slot. Return null if `index` is out of the bound of
+ * `[0, objectSize())`. `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get
+ * @return the Objectfield at the `index` slot, or null if `index` is out of bounds
+ */
+ public ObjectField getFieldAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int id = readUnsigned(value, idStart + idSize * index, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ String key = getMetadataKey(metadata, id);
+ Variant v = new Variant(value, metadata, dataStart + offset);
+ return new ObjectField(key, v);
+ });
+ }
+
+ /**
+ * Returns the dictionary ID for the object field at the `index` slot.
+ * `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get the dictionary ID for
+ * @return the dictionary ID for the object field at the `index` slot
+ * @throws MalformedVariantException if `index` is out of bounds
+ */
+ public int getDictionaryIdAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) {
+ throw malformedVariant();
+ }
+ return readUnsigned(value, idStart + idSize * index, idSize);
+ });
+ }
+
+ /**
+ * @return the number of array elements. `getType()` must be `Type.ARRAY`.
+ */
+ public int arraySize() {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> size);
+ }
+
+ /**
+ * Returns the array element Variant value at the `index` slot. Returns null if `index` is
+ * out of the bound of `[0, arraySize())`. `getType()` must be `Type.ARRAY`.
+ * @param index the index of the array element to get
+ * @return the array element Variant at the `index` slot, or null if `index` is out of bounds
+ */
+ public Variant getElementAtIndex(int index) {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ });
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId) {
+ return toJson(zoneId, false);
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @param truncateTrailingZeros Whether to truncate trailing zeros in decimal values or timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId, boolean truncateTrailingZeros) {
+ StringBuilder sb = new StringBuilder();
+ toJsonImpl(value, metadata, pos, sb, zoneId, truncateTrailingZeros);
+ return sb.toString();
+ }
+
+ /**
+ * Escapes a string so that it can be pasted into a JSON structure. For example, if `str`
+ * only contains a new-line character, then the result is "\n" (4 characters)
+ * @param str the string to escape
+ * @return the escaped string
+ */
+ private static String escapeJson(String str) {
+ try (CharArrayWriter writer = new CharArrayWriter();
+ JsonGenerator gen = new JsonFactory().createGenerator(writer)) {
+ gen.writeString(str);
+ gen.flush();
+ return writer.toString();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Appends a quoted string to a StringBuilder. It is used when we know `str` doesn't contain any
+ * special characters that needs escaping. This is more performant than
+ * `sb.append(escapeJson(str))`.
+ * @param sb the StringBuilder to append to
+ * @param str the string to append
+ */
+ private static void appendQuoted(StringBuilder sb, String str) {
+ sb.append('"');
+ sb.append(str);
+ sb.append('"');
+ }
+
+ /** The format for a timestamp without time zone. */
+ private static final DateTimeFormatter TIMESTAMP_NTZ_FORMATTER = new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .appendLiteral('T')
+ .appendPattern("HH:mm:ss")
+ .appendFraction(MICRO_OF_SECOND, 6, 6, true)
+ .toFormatter(Locale.US);
+
+ /** The format for a timestamp with time zone. */
+ private static final DateTimeFormatter TIMESTAMP_FORMATTER = new DateTimeFormatterBuilder()
+ .append(TIMESTAMP_NTZ_FORMATTER)
+ .appendOffset("+HH:MM", "+00:00")
+ .toFormatter(Locale.US);
+
+ /** The format for a timestamp without time zone, truncating trailing microsecond zeros. */
+ private static final DateTimeFormatter TIMESTAMP_NTZ_TRUNC_FORMATTER = new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .appendLiteral('T')
+ .appendPattern("HH:mm:ss")
+ .optionalStart()
+ .appendFraction(MICRO_OF_SECOND, 0, 6, true)
+ .optionalEnd()
+ .toFormatter(Locale.US);
+
+ /** The format for a timestamp with time zone, truncating trailing microsecond zeros. */
+ private static final DateTimeFormatter TIMESTAMP_TRUNC_FORMATTER = new DateTimeFormatterBuilder()
+ .append(TIMESTAMP_NTZ_TRUNC_FORMATTER)
+ .appendOffset("+HH:MM", "+00:00")
+ .toFormatter(Locale.US);
+
+ private static Instant microsToInstant(long microsSinceEpoch) {
+ return Instant.EPOCH.plus(microsSinceEpoch, ChronoUnit.MICROS);
+ }
+
+ private static void toJsonImpl(
+ byte[] value, byte[] metadata, int pos, StringBuilder sb, ZoneId zoneId, boolean truncateTrailingZeros) {
Review Comment:
Switched to using the jackson generator.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ // Use unsigned right shift to compute the middle of `low` and `high`. This is not only a
+ // performance optimization, because it can properly handle the case where `low + high`
+ // overflows int.
+ int mid = (low + high) >>> 1;
+ int id = readUnsigned(value, idStart + idSize * mid, idSize);
+ int cmp = getMetadataKey(metadata, id).compareTo(key);
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ int offset = readUnsigned(value, offsetStart + offsetSize * mid, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ }
+ return null;
+ });
+ }
+
+ /**
+ * A field in a Variant object.
+ */
+ public static final class ObjectField {
+ public final String key;
+ public final Variant value;
+
+ public ObjectField(String key, Variant value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Get the object field at the `index` slot. Return null if `index` is out of the bound of
+ // `[0, objectSize())`.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+ /**
+ * Returns the object field at the `index` slot. Return null if `index` is out of the bound of
+ * `[0, objectSize())`. `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get
+ * @return the Objectfield at the `index` slot, or null if `index` is out of bounds
+ */
+ public ObjectField getFieldAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int id = readUnsigned(value, idStart + idSize * index, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ String key = getMetadataKey(metadata, id);
+ Variant v = new Variant(value, metadata, dataStart + offset);
+ return new ObjectField(key, v);
+ });
+ }
+
+ /**
+ * Returns the dictionary ID for the object field at the `index` slot.
+ * `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get the dictionary ID for
+ * @return the dictionary ID for the object field at the `index` slot
+ * @throws MalformedVariantException if `index` is out of bounds
+ */
+ public int getDictionaryIdAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) {
+ throw malformedVariant();
+ }
+ return readUnsigned(value, idStart + idSize * index, idSize);
+ });
+ }
+
+ /**
+ * @return the number of array elements. `getType()` must be `Type.ARRAY`.
+ */
+ public int arraySize() {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> size);
+ }
+
+ /**
+ * Returns the array element Variant value at the `index` slot. Returns null if `index` is
+ * out of the bound of `[0, arraySize())`. `getType()` must be `Type.ARRAY`.
+ * @param index the index of the array element to get
+ * @return the array element Variant at the `index` slot, or null if `index` is out of bounds
+ */
+ public Variant getElementAtIndex(int index) {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ });
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId) {
+ return toJson(zoneId, false);
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @param truncateTrailingZeros Whether to truncate trailing zeros in decimal values or timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId, boolean truncateTrailingZeros) {
+ StringBuilder sb = new StringBuilder();
+ toJsonImpl(value, metadata, pos, sb, zoneId, truncateTrailingZeros);
+ return sb.toString();
+ }
+
+ /**
+ * Escapes a string so that it can be pasted into a JSON structure. For example, if `str`
+ * only contains a new-line character, then the result is "\n" (4 characters)
+ * @param str the string to escape
+ * @return the escaped string
+ */
+ private static String escapeJson(String str) {
+ try (CharArrayWriter writer = new CharArrayWriter();
+ JsonGenerator gen = new JsonFactory().createGenerator(writer)) {
+ gen.writeString(str);
+ gen.flush();
+ return writer.toString();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Appends a quoted string to a StringBuilder. It is used when we know `str` doesn't contain any
+ * special characters that needs escaping. This is more performant than
+ * `sb.append(escapeJson(str))`.
+ * @param sb the StringBuilder to append to
+ * @param str the string to append
+ */
+ private static void appendQuoted(StringBuilder sb, String str) {
+ sb.append('"');
+ sb.append(str);
+ sb.append('"');
+ }
+
+ /** The format for a timestamp without time zone. */
+ private static final DateTimeFormatter TIMESTAMP_NTZ_FORMATTER = new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .appendLiteral('T')
+ .appendPattern("HH:mm:ss")
+ .appendFraction(MICRO_OF_SECOND, 6, 6, true)
+ .toFormatter(Locale.US);
+
+ /** The format for a timestamp with time zone. */
+ private static final DateTimeFormatter TIMESTAMP_FORMATTER = new DateTimeFormatterBuilder()
+ .append(TIMESTAMP_NTZ_FORMATTER)
+ .appendOffset("+HH:MM", "+00:00")
+ .toFormatter(Locale.US);
+
+ /** The format for a timestamp without time zone, truncating trailing microsecond zeros. */
+ private static final DateTimeFormatter TIMESTAMP_NTZ_TRUNC_FORMATTER = new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .appendLiteral('T')
+ .appendPattern("HH:mm:ss")
+ .optionalStart()
+ .appendFraction(MICRO_OF_SECOND, 0, 6, true)
+ .optionalEnd()
+ .toFormatter(Locale.US);
+
+ /** The format for a timestamp with time zone, truncating trailing microsecond zeros. */
+ private static final DateTimeFormatter TIMESTAMP_TRUNC_FORMATTER = new DateTimeFormatterBuilder()
+ .append(TIMESTAMP_NTZ_TRUNC_FORMATTER)
+ .appendOffset("+HH:MM", "+00:00")
+ .toFormatter(Locale.US);
+
+ private static Instant microsToInstant(long microsSinceEpoch) {
+ return Instant.EPOCH.plus(microsSinceEpoch, ChronoUnit.MICROS);
+ }
+
+ private static void toJsonImpl(
+ byte[] value, byte[] metadata, int pos, StringBuilder sb, ZoneId zoneId, boolean truncateTrailingZeros) {
+ switch (VariantUtil.getType(value, pos)) {
+ case OBJECT:
+ handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ sb.append('{');
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
Review Comment:
I don't think I fully understand your suggestion, but I did simplify this in order to avoid a lot of "similar" code.
##########
parquet-variant/pom.xml:
##########
@@ -0,0 +1,88 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet</artifactId>
+ <relativePath>../pom.xml</relativePath>
+ <version>1.16.0-SNAPSHOT</version>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>parquet-variant</artifactId>
+ <packaging>jar</packaging>
+
+ <name>Apache Parquet Variant</name>
+ <url>https://parquet.apache.org</url>
+
+ <properties>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-jackson</artifactId>
+ <version>${project.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>${jackson.groupId}</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${jackson.groupId}</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson-databind.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
Review Comment:
I ended up removing this dependency.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java:
##########
@@ -0,0 +1,631 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
Review Comment:
Removed.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+import static java.time.temporal.ChronoField.*;
+import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos,
+ (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
Review Comment:
Moved to class.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ // Use unsigned right shift to compute the middle of `low` and `high`. This is not only a
+ // performance optimization, because it can properly handle the case where `low + high`
+ // overflows int.
+ int mid = (low + high) >>> 1;
+ int id = readUnsigned(value, idStart + idSize * mid, idSize);
+ int cmp = getMetadataKey(metadata, id).compareTo(key);
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ int offset = readUnsigned(value, offsetStart + offsetSize * mid, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ }
+ return null;
+ });
+ }
+
+ /**
+ * A field in a Variant object.
+ */
+ public static final class ObjectField {
+ public final String key;
+ public final Variant value;
+
+ public ObjectField(String key, Variant value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Get the object field at the `index` slot. Return null if `index` is out of the bound of
+ // `[0, objectSize())`.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+ /**
+ * Returns the object field at the `index` slot. Return null if `index` is out of the bound of
+ * `[0, objectSize())`. `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get
+ * @return the Objectfield at the `index` slot, or null if `index` is out of bounds
+ */
+ public ObjectField getFieldAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int id = readUnsigned(value, idStart + idSize * index, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ String key = getMetadataKey(metadata, id);
+ Variant v = new Variant(value, metadata, dataStart + offset);
+ return new ObjectField(key, v);
+ });
+ }
+
+ /**
+ * Returns the dictionary ID for the object field at the `index` slot.
+ * `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get the dictionary ID for
+ * @return the dictionary ID for the object field at the `index` slot
+ * @throws MalformedVariantException if `index` is out of bounds
+ */
+ public int getDictionaryIdAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) {
+ throw malformedVariant();
+ }
+ return readUnsigned(value, idStart + idSize * index, idSize);
+ });
+ }
+
+ /**
+ * @return the number of array elements. `getType()` must be `Type.ARRAY`.
+ */
+ public int arraySize() {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> size);
+ }
+
+ /**
+ * Returns the array element Variant value at the `index` slot. Returns null if `index` is
+ * out of the bound of `[0, arraySize())`. `getType()` must be `Type.ARRAY`.
+ * @param index the index of the array element to get
+ * @return the array element Variant at the `index` slot, or null if `index` is out of bounds
+ */
+ public Variant getElementAtIndex(int index) {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ });
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId) {
Review Comment:
I added the `toJson()` which defaults to `+00:00`. The options are there for engines to choose the behavior, while sharing the same implementation.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
Review Comment:
We want to multiply `idSize * i`, so it has to stay inside the loop.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,435 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+import static java.time.temporal.ChronoField.*;
+import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos,
+ (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ // Use unsigned right shift to compute the middle of `low` and `high`. This is not only a
+ // performance optimization, because it can properly handle the case where `low + high`
+ // overflows int.
+ int mid = (low + high) >>> 1;
+ int id = readUnsigned(value, idStart + idSize * mid, idSize);
+ int cmp = getMetadataKey(metadata, id).compareTo(key);
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ int offset = readUnsigned(value, offsetStart + offsetSize * mid, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ }
+ return null;
+ });
+ }
+
+ /**
+ * A field in a Variant object.
+ */
+ public static final class ObjectField {
+ public final String key;
+ public final Variant value;
+
+ public ObjectField(String key, Variant value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Get the object field at the `index` slot. Return null if `index` is out of the bound of
+ // `[0, objectSize())`.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+ /**
+ * Returns the object field at the `index` slot. Return null if `index` is out of the bound of
+ * `[0, objectSize())`. `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get
+ * @return the Objectfield at the `index` slot, or null if `index` is out of bounds
+ */
+ public ObjectField getFieldAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int id = readUnsigned(value, idStart + idSize * index, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ String key = getMetadataKey(metadata, id);
+ Variant v = new Variant(value, metadata, dataStart + offset);
+ return new ObjectField(key, v);
+ });
+ }
+
+ /**
+ * Returns the dictionary ID for the object field at the `index` slot.
+ * `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get the dictionary ID for
+ * @return the dictionary ID for the object field at the `index` slot
+ * @throws MalformedVariantException if `index` is out of bounds
+ */
+ public int getDictionaryIdAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) {
+ throw malformedVariant();
+ }
+ return readUnsigned(value, idStart + idSize * index, idSize);
+ });
+ }
+
+ /**
+ * @return the number of array elements. `getType()` must be `Type.ARRAY`.
+ */
+ public int arraySize() {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> size);
+ }
+
+ /**
+ * Returns the array element Variant value at the `index` slot. Returns null if `index` is
+ * out of the bound of `[0, arraySize())`. `getType()` must be `Type.ARRAY`.
+ * @param index the index of the array element to get
+ * @return the array element Variant at the `index` slot, or null if `index` is out of bounds
+ */
+ public Variant getElementAtIndex(int index) {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ });
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId) {
+ return toJson(zoneId, false);
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @param truncateTrailingZeros Whether to truncate trailing zeros in decimal values or timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId, boolean truncateTrailingZeros) {
+ StringBuilder sb = new StringBuilder();
+ toJsonImpl(value, metadata, pos, sb, zoneId, truncateTrailingZeros);
+ return sb.toString();
+ }
+
+ /**
+ * Escapes a string so that it can be pasted into a JSON structure. For example, if `str`
+ * only contains a new-line character, then the result is "\n" (4 characters)
+ * @param str the string to escape
+ * @return the escaped string
+ */
Review Comment:
Added.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java:
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.exc.InputCoercionException;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+/**
+ * Builder for creating Variant value and metadata.
+ */
+public class VariantBuilder {
+ public VariantBuilder(boolean allowDuplicateKeys) {
Review Comment:
This is not for writing duplicate keys in the Variant value itself, but for parsing JSON strings. JSON strings might have duplicate keys, and this flag controls the behavior when encountering duplicate keys.
I added a comment to clarify.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java:
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.exc.InputCoercionException;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+/**
+ * Builder for creating Variant value and metadata.
+ */
+public class VariantBuilder {
+ public VariantBuilder(boolean allowDuplicateKeys) {
+ this(allowDuplicateKeys, DEFAULT_SIZE_LIMIT);
+ }
+
+ public VariantBuilder(boolean allowDuplicateKeys, int sizeLimitBytes) {
+ this.allowDuplicateKeys = allowDuplicateKeys;
+ this.sizeLimitBytes = sizeLimitBytes;
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json) throws IOException {
+ return parseJson(json, new VariantBuilder(false));
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json, VariantBuilder builder) throws IOException {
+ try (JsonParser parser = new JsonFactory().createParser(json)) {
+ parser.nextToken();
+ return parseJson(parser, builder);
+ }
+ }
+
+ /**
+ * Parse a JSON parser as a Variant value.
+ * @param parser the JSON parser to use
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(JsonParser parser, VariantBuilder builder) throws IOException {
+ builder.buildFromJsonParser(parser);
+ return builder.result();
+ }
+
+ /**
+ * @return the Variant value
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public Variant result() {
+ int numKeys = dictionaryKeys.size();
+ // Use long to avoid overflow in accumulating lengths.
+ long dictionaryStringSize = 0;
+ for (byte[] key : dictionaryKeys) {
+ dictionaryStringSize += key.length;
+ }
+ // Determine the number of bytes required per offset entry.
+ // The largest offset is the one-past-the-end value, which is total string size. It's very
+ // unlikely that the number of keys could be larger, but incorporate that into the calculation
+ // in case of pathological data.
+ long maxSize = Math.max(dictionaryStringSize, numKeys);
+ if (maxSize > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ int offsetSize = getMinIntegerSize((int) maxSize);
+
+ int offsetStart = 1 + offsetSize;
+ int stringStart = offsetStart + (numKeys + 1) * offsetSize;
+ long metadataSize = stringStart + dictionaryStringSize;
+
+ if (metadataSize > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ byte[] metadata = new byte[(int) metadataSize];
+ int headerByte = VERSION | ((offsetSize - 1) << 6);
+ writeLong(metadata, 0, headerByte, 1);
+ writeLong(metadata, 1, numKeys, offsetSize);
+ int currentOffset = 0;
+ for (int i = 0; i < numKeys; ++i) {
+ writeLong(metadata, offsetStart + i * offsetSize, currentOffset, offsetSize);
+ byte[] key = dictionaryKeys.get(i);
+ System.arraycopy(key, 0, metadata, stringStart + currentOffset, key.length);
+ currentOffset += key.length;
+ }
+ writeLong(metadata, offsetStart + numKeys * offsetSize, currentOffset, offsetSize);
+ return new Variant(Arrays.copyOfRange(writeBuffer, 0, writePos), metadata);
+ }
+
+ public void appendString(String str) {
+ byte[] text = str.getBytes(StandardCharsets.UTF_8);
+ boolean longStr = text.length > MAX_SHORT_STR_SIZE;
+ checkCapacity((longStr ? 1 + U32_SIZE : 1) + text.length);
+ if (longStr) {
+ writeBuffer[writePos++] = primitiveHeader(LONG_STR);
+ writeLong(writeBuffer, writePos, text.length, U32_SIZE);
+ writePos += U32_SIZE;
+ } else {
+ writeBuffer[writePos++] = shortStrHeader(text.length);
+ }
+ System.arraycopy(text, 0, writeBuffer, writePos, text.length);
+ writePos += text.length;
+ }
+
+ public void appendNull() {
+ checkCapacity(1);
+ writeBuffer[writePos++] = primitiveHeader(NULL);
+ }
+
+ public void appendBoolean(boolean b) {
+ checkCapacity(1);
+ writeBuffer[writePos++] = primitiveHeader(b ? TRUE : FALSE);
+ }
+
+ /**
+ * Appends a long value to the variant builder. The actual encoded integer type depends on the
+ * value range of the long value.
+ * @param l the long value to append
+ */
+ public void appendLong(long l) {
+ checkCapacity(1 + 8);
+ if (l == (byte) l) {
+ writeBuffer[writePos++] = primitiveHeader(INT1);
+ writeLong(writeBuffer, writePos, l, 1);
+ writePos += 1;
+ } else if (l == (short) l) {
+ writeBuffer[writePos++] = primitiveHeader(INT2);
+ writeLong(writeBuffer, writePos, l, 2);
+ writePos += 2;
+ } else if (l == (int) l) {
+ writeBuffer[writePos++] = primitiveHeader(INT4);
+ writeLong(writeBuffer, writePos, l, 4);
+ writePos += 4;
+ } else {
+ writeBuffer[writePos++] = primitiveHeader(INT8);
+ writeLong(writeBuffer, writePos, l, 8);
+ writePos += 8;
+ }
+ }
+
+ public void appendDouble(double d) {
+ checkCapacity(1 + 8);
+ writeBuffer[writePos++] = primitiveHeader(DOUBLE);
+ writeLong(writeBuffer, writePos, Double.doubleToLongBits(d), 8);
+ writePos += 8;
+ }
+
+ /**
+ * Appends a decimal value to the variant builder. The actual encoded decimal type depends on the
+ * precision and scale of the decimal value.
+ * @param d the decimal value to append
+ */
+ public void appendDecimal(BigDecimal d) {
+ checkCapacity(2 + 16);
+ BigInteger unscaled = d.unscaledValue();
+ if (d.scale() <= MAX_DECIMAL4_PRECISION && d.precision() <= MAX_DECIMAL4_PRECISION) {
+ writeBuffer[writePos++] = primitiveHeader(DECIMAL4);
+ writeBuffer[writePos++] = (byte) d.scale();
+ writeLong(writeBuffer, writePos, unscaled.intValueExact(), 4);
+ writePos += 4;
+ } else if (d.scale() <= MAX_DECIMAL8_PRECISION && d.precision() <= MAX_DECIMAL8_PRECISION) {
+ writeBuffer[writePos++] = primitiveHeader(DECIMAL8);
+ writeBuffer[writePos++] = (byte) d.scale();
+ writeLong(writeBuffer, writePos, unscaled.longValueExact(), 8);
+ writePos += 8;
+ } else {
+ assert d.scale() <= MAX_DECIMAL16_PRECISION && d.precision() <= MAX_DECIMAL16_PRECISION;
+ writeBuffer[writePos++] = primitiveHeader(DECIMAL16);
+ writeBuffer[writePos++] = (byte) d.scale();
+ // `toByteArray` returns a big-endian representation. We need to copy it reversely and sign
+ // extend it to 16 bytes.
+ byte[] bytes = unscaled.toByteArray();
+ for (int i = 0; i < bytes.length; ++i) {
+ writeBuffer[writePos + i] = bytes[bytes.length - 1 - i];
+ }
+ byte sign = (byte) (bytes[0] < 0 ? -1 : 0);
+ for (int i = bytes.length; i < 16; ++i) {
+ writeBuffer[writePos + i] = sign;
+ }
+ writePos += 16;
+ }
+ }
+
+ public void appendDate(int daysSinceEpoch) {
+ checkCapacity(1 + 4);
+ writeBuffer[writePos++] = primitiveHeader(DATE);
+ writeLong(writeBuffer, writePos, daysSinceEpoch, 4);
+ writePos += 4;
+ }
+
+ public void appendTimestamp(long microsSinceEpoch) {
+ checkCapacity(1 + 8);
+ writeBuffer[writePos++] = primitiveHeader(TIMESTAMP);
+ writeLong(writeBuffer, writePos, microsSinceEpoch, 8);
+ writePos += 8;
+ }
+
+ public void appendTimestampNtz(long microsSinceEpoch) {
+ checkCapacity(1 + 8);
+ writeBuffer[writePos++] = primitiveHeader(TIMESTAMP_NTZ);
+ writeLong(writeBuffer, writePos, microsSinceEpoch, 8);
+ writePos += 8;
+ }
+
+ public void appendFloat(float f) {
+ checkCapacity(1 + 4);
+ writeBuffer[writePos++] = primitiveHeader(FLOAT);
+ writeLong(writeBuffer, writePos, Float.floatToIntBits(f), 8);
+ writePos += 4;
+ }
+
+ public void appendBinary(byte[] binary) {
+ checkCapacity(1 + U32_SIZE + binary.length);
+ writeBuffer[writePos++] = primitiveHeader(BINARY);
+ writeLong(writeBuffer, writePos, binary.length, U32_SIZE);
+ writePos += U32_SIZE;
+ System.arraycopy(binary, 0, writeBuffer, writePos, binary.length);
+ writePos += binary.length;
+ }
+
+ /**
+ * Adds a key to the Variant dictionary. If the key already exists, the dictionary is unmodified.
+ * @param key the key to add
+ * @return the id of the key
+ */
+ public int addKey(String key) {
+ int id;
+ if (dictionary.containsKey(key)) {
+ id = dictionary.get(key);
+ } else {
+ id = dictionaryKeys.size();
+ dictionary.put(key, id);
+ dictionaryKeys.add(key.getBytes(StandardCharsets.UTF_8));
+ }
+ return id;
+ }
Review Comment:
Done, thanks!
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
Review Comment:
Added comment and braces.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ // Use unsigned right shift to compute the middle of `low` and `high`. This is not only a
+ // performance optimization, because it can properly handle the case where `low + high`
+ // overflows int.
+ int mid = (low + high) >>> 1;
+ int id = readUnsigned(value, idStart + idSize * mid, idSize);
+ int cmp = getMetadataKey(metadata, id).compareTo(key);
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ int offset = readUnsigned(value, offsetStart + offsetSize * mid, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ }
+ return null;
+ });
+ }
+
+ /**
+ * A field in a Variant object.
+ */
+ public static final class ObjectField {
+ public final String key;
+ public final Variant value;
+
+ public ObjectField(String key, Variant value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Get the object field at the `index` slot. Return null if `index` is out of the bound of
+ // `[0, objectSize())`.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+ /**
+ * Returns the object field at the `index` slot. Return null if `index` is out of the bound of
+ * `[0, objectSize())`. `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get
+ * @return the Objectfield at the `index` slot, or null if `index` is out of bounds
Review Comment:
Done
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
Review Comment:
Where do we assume that `metadata` and `value` are in the same array? I don't think we are making that assumption.
The `pos` part in `getValue()` is not assuming the metadata is in the same array, but is for getting a "sub-variant" value from a variant value.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ // Use unsigned right shift to compute the middle of `low` and `high`. This is not only a
+ // performance optimization, because it can properly handle the case where `low + high`
+ // overflows int.
+ int mid = (low + high) >>> 1;
+ int id = readUnsigned(value, idStart + idSize * mid, idSize);
+ int cmp = getMetadataKey(metadata, id).compareTo(key);
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ int offset = readUnsigned(value, offsetStart + offsetSize * mid, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ }
+ return null;
+ });
+ }
+
+ /**
+ * A field in a Variant object.
+ */
+ public static final class ObjectField {
+ public final String key;
+ public final Variant value;
+
+ public ObjectField(String key, Variant value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Get the object field at the `index` slot. Return null if `index` is out of the bound of
+ // `[0, objectSize())`.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+ /**
+ * Returns the object field at the `index` slot. Return null if `index` is out of the bound of
+ * `[0, objectSize())`. `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get
+ * @return the Objectfield at the `index` slot, or null if `index` is out of bounds
+ */
+ public ObjectField getFieldAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int id = readUnsigned(value, idStart + idSize * index, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ String key = getMetadataKey(metadata, id);
+ Variant v = new Variant(value, metadata, dataStart + offset);
+ return new ObjectField(key, v);
+ });
+ }
+
+ /**
+ * Returns the dictionary ID for the object field at the `index` slot.
+ * `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get the dictionary ID for
+ * @return the dictionary ID for the object field at the `index` slot
+ * @throws MalformedVariantException if `index` is out of bounds
+ */
+ public int getDictionaryIdAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) {
+ throw malformedVariant();
+ }
+ return readUnsigned(value, idStart + idSize * index, idSize);
+ });
+ }
+
+ /**
+ * @return the number of array elements. `getType()` must be `Type.ARRAY`.
+ */
+ public int arraySize() {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> size);
+ }
+
+ /**
+ * Returns the array element Variant value at the `index` slot. Returns null if `index` is
+ * out of the bound of `[0, arraySize())`. `getType()` must be `Type.ARRAY`.
+ * @param index the index of the array element to get
+ * @return the array element Variant at the `index` slot, or null if `index` is out of bounds
+ */
+ public Variant getElementAtIndex(int index) {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ });
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId) {
+ return toJson(zoneId, false);
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @param truncateTrailingZeros Whether to truncate trailing zeros in decimal values or timestamps
Review Comment:
This is an option that engines can choose, while not having to reimplement all the Variant-navigation code.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java:
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.exc.InputCoercionException;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+/**
+ * Builder for creating Variant value and metadata.
+ */
+public class VariantBuilder {
+ public VariantBuilder(boolean allowDuplicateKeys) {
+ this(allowDuplicateKeys, DEFAULT_SIZE_LIMIT);
+ }
+
+ public VariantBuilder(boolean allowDuplicateKeys, int sizeLimitBytes) {
+ this.allowDuplicateKeys = allowDuplicateKeys;
+ this.sizeLimitBytes = sizeLimitBytes;
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json) throws IOException {
+ return parseJson(json, new VariantBuilder(false));
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json, VariantBuilder builder) throws IOException {
+ try (JsonParser parser = new JsonFactory().createParser(json)) {
+ parser.nextToken();
+ return parseJson(parser, builder);
+ }
+ }
+
+ /**
+ * Parse a JSON parser as a Variant value.
+ * @param parser the JSON parser to use
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(JsonParser parser, VariantBuilder builder) throws IOException {
+ builder.buildFromJsonParser(parser);
+ return builder.result();
+ }
+
+ /**
+ * @return the Variant value
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public Variant result() {
+ int numKeys = dictionaryKeys.size();
+ // Use long to avoid overflow in accumulating lengths.
+ long dictionaryStringSize = 0;
+ for (byte[] key : dictionaryKeys) {
+ dictionaryStringSize += key.length;
+ }
+ // Determine the number of bytes required per offset entry.
+ // The largest offset is the one-past-the-end value, which is total string size. It's very
+ // unlikely that the number of keys could be larger, but incorporate that into the calculation
+ // in case of pathological data.
+ long maxSize = Math.max(dictionaryStringSize, numKeys);
+ if (maxSize > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ int offsetSize = getMinIntegerSize((int) maxSize);
+
+ int offsetStart = 1 + offsetSize;
+ int stringStart = offsetStart + (numKeys + 1) * offsetSize;
+ long metadataSize = stringStart + dictionaryStringSize;
+
+ if (metadataSize > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ byte[] metadata = new byte[(int) metadataSize];
+ int headerByte = VERSION | ((offsetSize - 1) << 6);
+ writeLong(metadata, 0, headerByte, 1);
+ writeLong(metadata, 1, numKeys, offsetSize);
+ int currentOffset = 0;
+ for (int i = 0; i < numKeys; ++i) {
+ writeLong(metadata, offsetStart + i * offsetSize, currentOffset, offsetSize);
+ byte[] key = dictionaryKeys.get(i);
+ System.arraycopy(key, 0, metadata, stringStart + currentOffset, key.length);
+ currentOffset += key.length;
+ }
+ writeLong(metadata, offsetStart + numKeys * offsetSize, currentOffset, offsetSize);
+ return new Variant(Arrays.copyOfRange(writeBuffer, 0, writePos), metadata);
+ }
+
+ public void appendString(String str) {
+ byte[] text = str.getBytes(StandardCharsets.UTF_8);
+ boolean longStr = text.length > MAX_SHORT_STR_SIZE;
+ checkCapacity((longStr ? 1 + U32_SIZE : 1) + text.length);
+ if (longStr) {
+ writeBuffer[writePos++] = primitiveHeader(LONG_STR);
+ writeLong(writeBuffer, writePos, text.length, U32_SIZE);
+ writePos += U32_SIZE;
+ } else {
+ writeBuffer[writePos++] = shortStrHeader(text.length);
+ }
+ System.arraycopy(text, 0, writeBuffer, writePos, text.length);
+ writePos += text.length;
+ }
+
+ public void appendNull() {
+ checkCapacity(1);
+ writeBuffer[writePos++] = primitiveHeader(NULL);
+ }
+
+ public void appendBoolean(boolean b) {
+ checkCapacity(1);
+ writeBuffer[writePos++] = primitiveHeader(b ? TRUE : FALSE);
+ }
+
+ /**
+ * Appends a long value to the variant builder. The actual encoded integer type depends on the
+ * value range of the long value.
+ * @param l the long value to append
+ */
+ public void appendLong(long l) {
+ checkCapacity(1 + 8);
Review Comment:
Updated to check based on what we write. We check here because we usually need to write some initial byte(s) before we write a long.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
Review Comment:
Renamed to "primitive type id"
##########
parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java:
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.exc.InputCoercionException;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+/**
+ * Builder for creating Variant value and metadata.
+ */
+public class VariantBuilder {
+ public VariantBuilder(boolean allowDuplicateKeys) {
+ this(allowDuplicateKeys, DEFAULT_SIZE_LIMIT);
+ }
+
+ public VariantBuilder(boolean allowDuplicateKeys, int sizeLimitBytes) {
+ this.allowDuplicateKeys = allowDuplicateKeys;
+ this.sizeLimitBytes = sizeLimitBytes;
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json) throws IOException {
+ return parseJson(json, new VariantBuilder(false));
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json, VariantBuilder builder) throws IOException {
+ try (JsonParser parser = new JsonFactory().createParser(json)) {
+ parser.nextToken();
+ return parseJson(parser, builder);
+ }
+ }
+
+ /**
+ * Parse a JSON parser as a Variant value.
+ * @param parser the JSON parser to use
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(JsonParser parser, VariantBuilder builder) throws IOException {
+ builder.buildFromJsonParser(parser);
+ return builder.result();
+ }
+
+ /**
+ * @return the Variant value
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public Variant result() {
+ int numKeys = dictionaryKeys.size();
+ // Use long to avoid overflow in accumulating lengths.
+ long dictionaryStringSize = 0;
+ for (byte[] key : dictionaryKeys) {
+ dictionaryStringSize += key.length;
+ }
+ // Determine the number of bytes required per offset entry.
+ // The largest offset is the one-past-the-end value, which is total string size. It's very
+ // unlikely that the number of keys could be larger, but incorporate that into the calculation
+ // in case of pathological data.
+ long maxSize = Math.max(dictionaryStringSize, numKeys);
+ if (maxSize > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ int offsetSize = getMinIntegerSize((int) maxSize);
+
+ int offsetStart = 1 + offsetSize;
+ int stringStart = offsetStart + (numKeys + 1) * offsetSize;
+ long metadataSize = stringStart + dictionaryStringSize;
+
+ if (metadataSize > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ byte[] metadata = new byte[(int) metadataSize];
+ int headerByte = VERSION | ((offsetSize - 1) << 6);
+ writeLong(metadata, 0, headerByte, 1);
+ writeLong(metadata, 1, numKeys, offsetSize);
+ int currentOffset = 0;
+ for (int i = 0; i < numKeys; ++i) {
+ writeLong(metadata, offsetStart + i * offsetSize, currentOffset, offsetSize);
+ byte[] key = dictionaryKeys.get(i);
+ System.arraycopy(key, 0, metadata, stringStart + currentOffset, key.length);
+ currentOffset += key.length;
+ }
+ writeLong(metadata, offsetStart + numKeys * offsetSize, currentOffset, offsetSize);
+ return new Variant(Arrays.copyOfRange(writeBuffer, 0, writePos), metadata);
+ }
+
+ public void appendString(String str) {
+ byte[] text = str.getBytes(StandardCharsets.UTF_8);
+ boolean longStr = text.length > MAX_SHORT_STR_SIZE;
+ checkCapacity((longStr ? 1 + U32_SIZE : 1) + text.length);
+ if (longStr) {
+ writeBuffer[writePos++] = primitiveHeader(LONG_STR);
+ writeLong(writeBuffer, writePos, text.length, U32_SIZE);
+ writePos += U32_SIZE;
+ } else {
+ writeBuffer[writePos++] = shortStrHeader(text.length);
+ }
+ System.arraycopy(text, 0, writeBuffer, writePos, text.length);
+ writePos += text.length;
+ }
+
+ public void appendNull() {
+ checkCapacity(1);
+ writeBuffer[writePos++] = primitiveHeader(NULL);
+ }
+
+ public void appendBoolean(boolean b) {
+ checkCapacity(1);
+ writeBuffer[writePos++] = primitiveHeader(b ? TRUE : FALSE);
+ }
+
+ /**
+ * Appends a long value to the variant builder. The actual encoded integer type depends on the
+ * value range of the long value.
+ * @param l the long value to append
+ */
+ public void appendLong(long l) {
+ checkCapacity(1 + 8);
+ if (l == (byte) l) {
+ writeBuffer[writePos++] = primitiveHeader(INT1);
+ writeLong(writeBuffer, writePos, l, 1);
+ writePos += 1;
+ } else if (l == (short) l) {
+ writeBuffer[writePos++] = primitiveHeader(INT2);
+ writeLong(writeBuffer, writePos, l, 2);
+ writePos += 2;
+ } else if (l == (int) l) {
+ writeBuffer[writePos++] = primitiveHeader(INT4);
+ writeLong(writeBuffer, writePos, l, 4);
+ writePos += 4;
+ } else {
+ writeBuffer[writePos++] = primitiveHeader(INT8);
+ writeLong(writeBuffer, writePos, l, 8);
+ writePos += 8;
+ }
+ }
+
+ public void appendDouble(double d) {
+ checkCapacity(1 + 8);
+ writeBuffer[writePos++] = primitiveHeader(DOUBLE);
+ writeLong(writeBuffer, writePos, Double.doubleToLongBits(d), 8);
+ writePos += 8;
+ }
+
+ /**
+ * Appends a decimal value to the variant builder. The actual encoded decimal type depends on the
+ * precision and scale of the decimal value.
+ * @param d the decimal value to append
+ */
+ public void appendDecimal(BigDecimal d) {
+ checkCapacity(2 + 16);
+ BigInteger unscaled = d.unscaledValue();
+ if (d.scale() <= MAX_DECIMAL4_PRECISION && d.precision() <= MAX_DECIMAL4_PRECISION) {
+ writeBuffer[writePos++] = primitiveHeader(DECIMAL4);
+ writeBuffer[writePos++] = (byte) d.scale();
+ writeLong(writeBuffer, writePos, unscaled.intValueExact(), 4);
+ writePos += 4;
+ } else if (d.scale() <= MAX_DECIMAL8_PRECISION && d.precision() <= MAX_DECIMAL8_PRECISION) {
+ writeBuffer[writePos++] = primitiveHeader(DECIMAL8);
+ writeBuffer[writePos++] = (byte) d.scale();
+ writeLong(writeBuffer, writePos, unscaled.longValueExact(), 8);
+ writePos += 8;
+ } else {
+ assert d.scale() <= MAX_DECIMAL16_PRECISION && d.precision() <= MAX_DECIMAL16_PRECISION;
+ writeBuffer[writePos++] = primitiveHeader(DECIMAL16);
+ writeBuffer[writePos++] = (byte) d.scale();
+ // `toByteArray` returns a big-endian representation. We need to copy it reversely and sign
+ // extend it to 16 bytes.
+ byte[] bytes = unscaled.toByteArray();
+ for (int i = 0; i < bytes.length; ++i) {
+ writeBuffer[writePos + i] = bytes[bytes.length - 1 - i];
+ }
+ byte sign = (byte) (bytes[0] < 0 ? -1 : 0);
+ for (int i = bytes.length; i < 16; ++i) {
+ writeBuffer[writePos + i] = sign;
+ }
+ writePos += 16;
+ }
+ }
+
+ public void appendDate(int daysSinceEpoch) {
+ checkCapacity(1 + 4);
+ writeBuffer[writePos++] = primitiveHeader(DATE);
+ writeLong(writeBuffer, writePos, daysSinceEpoch, 4);
+ writePos += 4;
+ }
+
+ public void appendTimestamp(long microsSinceEpoch) {
+ checkCapacity(1 + 8);
+ writeBuffer[writePos++] = primitiveHeader(TIMESTAMP);
+ writeLong(writeBuffer, writePos, microsSinceEpoch, 8);
+ writePos += 8;
+ }
+
+ public void appendTimestampNtz(long microsSinceEpoch) {
+ checkCapacity(1 + 8);
+ writeBuffer[writePos++] = primitiveHeader(TIMESTAMP_NTZ);
+ writeLong(writeBuffer, writePos, microsSinceEpoch, 8);
+ writePos += 8;
+ }
+
+ public void appendFloat(float f) {
+ checkCapacity(1 + 4);
+ writeBuffer[writePos++] = primitiveHeader(FLOAT);
+ writeLong(writeBuffer, writePos, Float.floatToIntBits(f), 8);
+ writePos += 4;
+ }
+
+ public void appendBinary(byte[] binary) {
+ checkCapacity(1 + U32_SIZE + binary.length);
+ writeBuffer[writePos++] = primitiveHeader(BINARY);
+ writeLong(writeBuffer, writePos, binary.length, U32_SIZE);
+ writePos += U32_SIZE;
+ System.arraycopy(binary, 0, writeBuffer, writePos, binary.length);
+ writePos += binary.length;
+ }
+
+ /**
+ * Adds a key to the Variant dictionary. If the key already exists, the dictionary is unmodified.
+ * @param key the key to add
+ * @return the id of the key
+ */
+ public int addKey(String key) {
+ int id;
+ if (dictionary.containsKey(key)) {
+ id = dictionary.get(key);
+ } else {
+ id = dictionaryKeys.size();
+ dictionary.put(key, id);
+ dictionaryKeys.add(key.getBytes(StandardCharsets.UTF_8));
+ }
+ return id;
+ }
+
+ /**
+ * @return the current write position of the variant builder
+ */
+ public int getWritePos() {
+ return writePos;
+ }
+
+ // Finish writing a variant object after all of its fields have already been written. The process
+ // is as follows:
+ // 1. The caller calls `getWritePos` before writing any fields to obtain the `start` parameter.
+ // 2. The caller appends all the object fields to the builder. In the meantime, it should maintain
+ // the `fields` parameter. Before appending each field, it should append an entry to `fields` to
+ // record the offset of the field. The offset is computed as `getWritePos() - start`.
+ // 3. The caller calls `finishWritingObject` to finish writing a variant object.
+ //
+ // This function is responsible to sort the fields by key. If there are duplicate field keys:
+ // - when `allowDuplicateKeys` is true, the field with the greatest offset value (the last
+ // appended one) is kept.
+ // - otherwise, throw an exception.
+ /**
+ * Finish writing a Variant object after all of its fields have already been written. The process
+ * is as follows:
+ * 1. The caller calls `getWritePos()` before writing any fields to obtain the `start` parameter.
+ * 2. The caller appends all the object fields to the builder. In the meantime, it should maintain
+ * the `fields` parameter. Before appending each field, it should append an entry to `fields` to
+ * record the offset of the field. The offset is computed as `getWritePos() - start`.
+ * 3. The caller calls `finishWritingObject` to finish writing the Variant object.
+ *
+ * This method will sort the fields by key. If there are duplicate field keys:
+ * - when `allowDuplicateKeys` is true, the field with the greatest offset value (the last
+ * appended one) is kept.
+ * - otherwise, throw an exception.
+ * @param start the start position of the object in the write buffer
+ * @param fields the list of `FieldEntry` in the object
+ * @throws VariantDuplicateKeyException if there are duplicate keys and `allowDuplicateKeys` is
+ * false
+ */
+ public void finishWritingObject(int start, ArrayList<FieldEntry> fields) {
+ int size = fields.size();
+ Collections.sort(fields);
+ int maxId = size == 0 ? 0 : fields.get(0).id;
+ if (allowDuplicateKeys) {
+ int distinctPos = 0;
+ // Maintain a list of distinct keys in-place.
+ for (int i = 1; i < size; ++i) {
+ maxId = Math.max(maxId, fields.get(i).id);
+ if (fields.get(i).id == fields.get(i - 1).id) {
+ // Found a duplicate key. Keep the field with the greater offset.
+ if (fields.get(distinctPos).offset < fields.get(i).offset) {
+ fields.set(distinctPos, fields.get(distinctPos).withNewOffset(fields.get(i).offset));
+ }
+ } else {
+ // Found a distinct key. Add the field to the list.
+ ++distinctPos;
+ fields.set(distinctPos, fields.get(i));
+ }
+ }
+ if (distinctPos + 1 < fields.size()) {
+ size = distinctPos + 1;
+ // Resize `fields` to `size`.
+ fields.subList(size, fields.size()).clear();
+ // Sort the fields by offsets so that we can move the value data of each field to the new
+ // offset without overwriting the fields after it.
+ fields.sort(Comparator.comparingInt(f -> f.offset));
+ int currentOffset = 0;
+ for (int i = 0; i < size; ++i) {
+ int oldOffset = fields.get(i).offset;
+ int fieldSize = VariantUtil.valueSize(writeBuffer, start + oldOffset);
+ System.arraycopy(writeBuffer, start + oldOffset, writeBuffer, start + currentOffset, fieldSize);
+ fields.set(i, fields.get(i).withNewOffset(currentOffset));
+ currentOffset += fieldSize;
+ }
+ writePos = start + currentOffset;
+ // Change back to the sort order by field keys, required by the Variant specification.
+ Collections.sort(fields);
+ }
+ } else {
+ for (int i = 1; i < size; ++i) {
+ maxId = Math.max(maxId, fields.get(i).id);
+ String key = fields.get(i).key;
+ if (key.equals(fields.get(i - 1).key)) {
+ throw new VariantDuplicateKeyException(key);
+ }
+ }
+ }
+ int dataSize = writePos - start;
+ boolean largeSize = size > U8_MAX;
+ int sizeBytes = largeSize ? U32_SIZE : 1;
+ int idSize = getMinIntegerSize(maxId);
+ int offsetSize = getMinIntegerSize(dataSize);
+ // The space for header byte, object size, id list, and offset list.
+ int headerSize = 1 + sizeBytes + size * idSize + (size + 1) * offsetSize;
+ checkCapacity(headerSize);
+ // Shift the just-written field data to make room for the object header section.
+ System.arraycopy(writeBuffer, start, writeBuffer, start + headerSize, dataSize);
+ writePos += headerSize;
+ writeBuffer[start] = objectHeader(largeSize, idSize, offsetSize);
+ writeLong(writeBuffer, start + 1, size, sizeBytes);
+ int idStart = start + 1 + sizeBytes;
+ int offsetStart = idStart + size * idSize;
+ for (int i = 0; i < size; ++i) {
+ writeLong(writeBuffer, idStart + i * idSize, fields.get(i).id, idSize);
+ writeLong(writeBuffer, offsetStart + i * offsetSize, fields.get(i).offset, offsetSize);
+ }
+ writeLong(writeBuffer, offsetStart + size * offsetSize, dataSize, offsetSize);
+ }
+
+ /**
+ * Finish writing a Variant array after all of its elements have already been written. The process
+ * is similar to that of `finishWritingObject`.
+ * @param start the start position of the array in the write buffer
+ * @param offsets the list of offsets of the array elements
+ */
+ public void finishWritingArray(int start, ArrayList<Integer> offsets) {
+ int dataSize = writePos - start;
+ int size = offsets.size();
+ boolean largeSize = size > U8_MAX;
+ int sizeBytes = largeSize ? U32_SIZE : 1;
+ int offsetSize = getMinIntegerSize(dataSize);
+ // The space for header byte, object size, and offset list.
+ int headerSize = 1 + sizeBytes + (size + 1) * offsetSize;
+ checkCapacity(headerSize);
+ // Shift the just-written field data to make room for the header section.
+ System.arraycopy(writeBuffer, start, writeBuffer, start + headerSize, dataSize);
+ writePos += headerSize;
+ writeBuffer[start] = arrayHeader(largeSize, offsetSize);
+ writeLong(writeBuffer, start + 1, size, sizeBytes);
+ int offsetStart = start + 1 + sizeBytes;
+ for (int i = 0; i < size; ++i) {
+ writeLong(writeBuffer, offsetStart + i * offsetSize, offsets.get(i), offsetSize);
+ }
+ writeLong(writeBuffer, offsetStart + size * offsetSize, dataSize, offsetSize);
+ }
+
+ /**
+ * Appends a Variant value to the Variant builder. The input Variant keys must be inserted into
+ * the builder dictionary and rebuilt with new field ids. For scalar values in the input
+ * Variant, we can directly copy the binary slice.
+ * @param v the Variant value to append
+ */
+ public void appendVariant(Variant v) {
+ appendVariantImpl(v.value, v.metadata, v.pos);
+ }
+
+ private void appendVariantImpl(byte[] value, byte[] metadata, int pos) {
+ checkIndex(pos, value.length);
+ int basicType = value[pos] & BASIC_TYPE_MASK;
+ switch (basicType) {
+ case OBJECT:
+ handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ ArrayList<FieldEntry> fields = new ArrayList<>(size);
+ int start = writePos;
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ int elementPos = dataStart + offset;
+ String key = getMetadataKey(metadata, id);
+ int newId = addKey(key);
+ fields.add(new FieldEntry(key, newId, writePos - start));
+ appendVariantImpl(value, metadata, elementPos);
+ }
+ finishWritingObject(start, fields);
+ return null;
+ });
+ break;
+ case ARRAY:
+ handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ ArrayList<Integer> offsets = new ArrayList<>(size);
+ int start = writePos;
+ for (int i = 0; i < size; ++i) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ int elementPos = dataStart + offset;
+ offsets.add(writePos - start);
+ appendVariantImpl(value, metadata, elementPos);
+ }
+ finishWritingArray(start, offsets);
+ return null;
+ });
+ break;
+ default:
+ shallowAppendVariantImpl(value, pos);
+ break;
+ }
+ }
+
+ private void shallowAppendVariantImpl(byte[] value, int pos) {
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ checkCapacity(size);
+ System.arraycopy(value, pos, writeBuffer, writePos, size);
+ writePos += size;
+ }
+
+ private void checkCapacity(int additionalBytes) {
+ int requiredBytes = writePos + additionalBytes;
+ if (requiredBytes > writeBuffer.length) {
+ // Allocate a new buffer with a capacity of the next power of 2 of `requiredBytes`.
+ int newCapacity = Integer.highestOneBit(requiredBytes);
+ newCapacity = newCapacity < requiredBytes ? newCapacity * 2 : newCapacity;
+ if (newCapacity > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ byte[] newValue = new byte[newCapacity];
+ System.arraycopy(writeBuffer, 0, newValue, 0, writePos);
+ writeBuffer = newValue;
+ }
+ }
+
+ // Temporarily store the information of a field. We need to collect all fields in an JSON object,
+ // sort them by their keys, and build the variant object in sorted order.
+
+ /**
+ * Class to store the information of a Variant object field. We need to collect all fields of
+ * an object, sort them by their keys, and build the Variant object in sorted order.
+ */
+ public static final class FieldEntry implements Comparable<FieldEntry> {
+ final String key;
+ final int id;
+ final int offset;
+
+ public FieldEntry(String key, int id, int offset) {
+ this.key = key;
+ this.id = id;
+ this.offset = offset;
+ }
+
+ FieldEntry withNewOffset(int newOffset) {
+ return new FieldEntry(key, id, newOffset);
+ }
+
+ @Override
+ public int compareTo(FieldEntry other) {
+ return key.compareTo(other.key);
+ }
+ }
+
+ private void buildFromJsonParser(JsonParser parser) throws IOException {
+ JsonToken token = parser.currentToken();
+ if (token == null) {
+ throw new JsonParseException(parser, "Unexpected null token");
+ }
+ switch (token) {
+ case START_OBJECT: {
+ ArrayList<FieldEntry> fields = new ArrayList<>();
+ int start = writePos;
+ while (parser.nextToken() != JsonToken.END_OBJECT) {
+ String key = parser.currentName();
+ parser.nextToken();
+ int id = addKey(key);
+ fields.add(new FieldEntry(key, id, writePos - start));
+ buildFromJsonParser(parser);
+ }
+ finishWritingObject(start, fields);
+ break;
+ }
+ case START_ARRAY: {
+ ArrayList<Integer> offsets = new ArrayList<>();
+ int start = writePos;
+ while (parser.nextToken() != JsonToken.END_ARRAY) {
+ offsets.add(writePos - start);
+ buildFromJsonParser(parser);
+ }
+ finishWritingArray(start, offsets);
+ break;
+ }
+ case VALUE_STRING:
+ appendString(parser.getText());
+ break;
+ case VALUE_NUMBER_INT:
+ try {
+ appendLong(parser.getLongValue());
+ } catch (InputCoercionException ignored) {
+ // If the value doesn't fit any integer type, parse it as decimal or floating instead.
+ parseAndAppendFloatingPoint(parser);
+ }
+ break;
+ case VALUE_NUMBER_FLOAT:
+ parseAndAppendFloatingPoint(parser);
+ break;
+ case VALUE_TRUE:
+ appendBoolean(true);
+ break;
+ case VALUE_FALSE:
+ appendBoolean(false);
+ break;
+ case VALUE_NULL:
+ appendNull();
+ break;
+ default:
+ throw new JsonParseException(parser, "Unexpected token " + token);
+ }
+ }
+
+ /**
+ * Returns the size (number of bytes) of the smallest unsigned integer type that can store
+ * `value`. It must be within `[0, U24_MAX]`.
+ * @param value the value to get the size for
+ * @return the size (number of bytes) of the smallest unsigned integer type that can store `value`
+ */
+ private int getMinIntegerSize(int value) {
+ assert value >= 0 && value <= U24_MAX;
+ if (value <= U8_MAX) return 1;
+ if (value <= U16_MAX) return 2;
+ return U24_SIZE;
Review Comment:
Done
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ // Use unsigned right shift to compute the middle of `low` and `high`. This is not only a
+ // performance optimization, because it can properly handle the case where `low + high`
+ // overflows int.
+ int mid = (low + high) >>> 1;
+ int id = readUnsigned(value, idStart + idSize * mid, idSize);
+ int cmp = getMetadataKey(metadata, id).compareTo(key);
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ int offset = readUnsigned(value, offsetStart + offsetSize * mid, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ }
+ return null;
+ });
+ }
+
+ /**
+ * A field in a Variant object.
+ */
+ public static final class ObjectField {
+ public final String key;
+ public final Variant value;
+
+ public ObjectField(String key, Variant value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Get the object field at the `index` slot. Return null if `index` is out of the bound of
+ // `[0, objectSize())`.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+ /**
+ * Returns the object field at the `index` slot. Return null if `index` is out of the bound of
+ * `[0, objectSize())`. `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get
+ * @return the Objectfield at the `index` slot, or null if `index` is out of bounds
+ */
+ public ObjectField getFieldAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int id = readUnsigned(value, idStart + idSize * index, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ String key = getMetadataKey(metadata, id);
+ Variant v = new Variant(value, metadata, dataStart + offset);
+ return new ObjectField(key, v);
+ });
+ }
+
+ /**
+ * Returns the dictionary ID for the object field at the `index` slot.
+ * `getType()` must be `Type.OBJECT`.
+ * @param index the index of the object field to get the dictionary ID for
+ * @return the dictionary ID for the object field at the `index` slot
+ * @throws MalformedVariantException if `index` is out of bounds
+ */
+ public int getDictionaryIdAtIndex(int index) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) {
+ throw malformedVariant();
+ }
+ return readUnsigned(value, idStart + idSize * index, idSize);
+ });
+ }
+
+ /**
+ * @return the number of array elements. `getType()` must be `Type.ARRAY`.
+ */
+ public int arraySize() {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> size);
+ }
+
+ /**
+ * Returns the array element Variant value at the `index` slot. Returns null if `index` is
+ * out of the bound of `[0, arraySize())`. `getType()` must be `Type.ARRAY`.
+ * @param index the index of the array element to get
+ * @return the array element Variant at the `index` slot, or null if `index` is out of bounds
+ */
+ public Variant getElementAtIndex(int index) {
+ return handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ if (index < 0 || index >= size) return null;
+ int offset = readUnsigned(value, offsetStart + offsetSize * index, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ });
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId) {
+ return toJson(zoneId, false);
+ }
+
+ /**
+ * @param zoneId The ZoneId to use for formatting timestamps
+ * @param truncateTrailingZeros Whether to truncate trailing zeros in decimal values or timestamps
+ * @return the JSON representation of the variant
+ * @throws MalformedVariantException if the variant is malformed
+ */
+ public String toJson(ZoneId zoneId, boolean truncateTrailingZeros) {
+ StringBuilder sb = new StringBuilder();
+ toJsonImpl(value, metadata, pos, sb, zoneId, truncateTrailingZeros);
+ return sb.toString();
+ }
+
+ /**
+ * Escapes a string so that it can be pasted into a JSON structure. For example, if `str`
+ * only contains a new-line character, then the result is "\n" (4 characters)
+ * @param str the string to escape
+ * @return the escaped string
+ */
+ private static String escapeJson(String str) {
+ try (CharArrayWriter writer = new CharArrayWriter();
+ JsonGenerator gen = new JsonFactory().createGenerator(writer)) {
+ gen.writeString(str);
+ gen.flush();
+ return writer.toString();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Appends a quoted string to a StringBuilder. It is used when we know `str` doesn't contain any
+ * special characters that needs escaping. This is more performant than
+ * `sb.append(escapeJson(str))`.
+ * @param sb the StringBuilder to append to
+ * @param str the string to append
+ */
+ private static void appendQuoted(StringBuilder sb, String str) {
+ sb.append('"');
+ sb.append(str);
+ sb.append('"');
+ }
+
+ /** The format for a timestamp without time zone. */
+ private static final DateTimeFormatter TIMESTAMP_NTZ_FORMATTER = new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .appendLiteral('T')
+ .appendPattern("HH:mm:ss")
+ .appendFraction(MICRO_OF_SECOND, 6, 6, true)
+ .toFormatter(Locale.US);
+
+ /** The format for a timestamp with time zone. */
+ private static final DateTimeFormatter TIMESTAMP_FORMATTER = new DateTimeFormatterBuilder()
+ .append(TIMESTAMP_NTZ_FORMATTER)
+ .appendOffset("+HH:MM", "+00:00")
+ .toFormatter(Locale.US);
+
+ /** The format for a timestamp without time zone, truncating trailing microsecond zeros. */
+ private static final DateTimeFormatter TIMESTAMP_NTZ_TRUNC_FORMATTER = new DateTimeFormatterBuilder()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE)
+ .appendLiteral('T')
+ .appendPattern("HH:mm:ss")
+ .optionalStart()
+ .appendFraction(MICRO_OF_SECOND, 0, 6, true)
+ .optionalEnd()
+ .toFormatter(Locale.US);
+
+ /** The format for a timestamp with time zone, truncating trailing microsecond zeros. */
+ private static final DateTimeFormatter TIMESTAMP_TRUNC_FORMATTER = new DateTimeFormatterBuilder()
+ .append(TIMESTAMP_NTZ_TRUNC_FORMATTER)
+ .appendOffset("+HH:MM", "+00:00")
+ .toFormatter(Locale.US);
+
+ private static Instant microsToInstant(long microsSinceEpoch) {
+ return Instant.EPOCH.plus(microsSinceEpoch, ChronoUnit.MICROS);
+ }
+
+ private static void toJsonImpl(
+ byte[] value, byte[] metadata, int pos, StringBuilder sb, ZoneId zoneId, boolean truncateTrailingZeros) {
+ switch (VariantUtil.getType(value, pos)) {
+ case OBJECT:
+ handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ sb.append('{');
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ int elementPos = dataStart + offset;
+ if (i != 0) sb.append(',');
+ sb.append(escapeJson(getMetadataKey(metadata, id)));
+ sb.append(':');
+ toJsonImpl(value, metadata, elementPos, sb, zoneId, truncateTrailingZeros);
+ }
+ sb.append('}');
+ return null;
+ });
+ break;
+ case ARRAY:
+ handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ sb.append('[');
+ for (int i = 0; i < size; ++i) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ int elementPos = dataStart + offset;
+ if (i != 0) sb.append(',');
+ toJsonImpl(value, metadata, elementPos, sb, zoneId, truncateTrailingZeros);
+ }
+ sb.append(']');
+ return null;
+ });
+ break;
+ case NULL:
+ sb.append("null");
+ break;
+ case BOOLEAN:
+ sb.append(VariantUtil.getBoolean(value, pos));
+ break;
+ case LONG:
+ sb.append(VariantUtil.getLong(value, pos));
+ break;
+ case STRING:
+ sb.append(escapeJson(VariantUtil.getString(value, pos)));
+ break;
+ case DOUBLE:
+ sb.append(VariantUtil.getDouble(value, pos));
+ break;
+ case DECIMAL:
+ if (truncateTrailingZeros) {
+ sb.append(VariantUtil.getDecimal(value, pos)
+ .stripTrailingZeros()
+ .toPlainString());
+ } else {
+ sb.append(VariantUtil.getDecimal(value, pos).toPlainString());
+ }
+ break;
+ case DATE:
+ appendQuoted(
+ sb,
+ LocalDate.ofEpochDay((int) VariantUtil.getLong(value, pos))
+ .toString());
+ break;
+ case TIMESTAMP:
+ if (truncateTrailingZeros) {
+ appendQuoted(
+ sb,
+ TIMESTAMP_TRUNC_FORMATTER.format(microsToInstant(VariantUtil.getLong(value, pos))
+ .atZone(zoneId)));
+ } else {
+ appendQuoted(
+ sb,
+ TIMESTAMP_FORMATTER.format(microsToInstant(VariantUtil.getLong(value, pos))
+ .atZone(zoneId)));
+ }
+ break;
+ case TIMESTAMP_NTZ:
+ if (truncateTrailingZeros) {
+ appendQuoted(
+ sb,
+ TIMESTAMP_NTZ_TRUNC_FORMATTER.format(microsToInstant(VariantUtil.getLong(value, pos))
+ .atZone(ZoneOffset.UTC)));
+ } else {
+ appendQuoted(
+ sb,
+ TIMESTAMP_NTZ_FORMATTER.format(microsToInstant(VariantUtil.getLong(value, pos))
+ .atZone(ZoneOffset.UTC)));
+ }
+ break;
+ case FLOAT:
+ sb.append(VariantUtil.getFloat(value, pos));
+ break;
+ case BINARY:
+ appendQuoted(sb, Base64.getEncoder().encodeToString(VariantUtil.getBinary(value, pos)));
+ break;
+ }
Review Comment:
Done
##########
parquet-variant/src/main/java/org/apache/parquet/variant/Variant.java:
##########
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import static java.time.temporal.ChronoField.*;
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Locale;
+
+/**
+ * This Variant class holds the Variant-encoded value and metadata binary values.
+ */
+public final class Variant {
+ final byte[] value;
+ final byte[] metadata;
+ /**
+ * The starting index into `value` where the variant value starts. This is used to avoid copying
+ * the value binary when reading a sub-variant in the array/object element.
+ */
+ final int pos;
+
+ public Variant(byte[] value, byte[] metadata) {
+ this(value, metadata, 0);
+ }
+
+ Variant(byte[] value, byte[] metadata, int pos) {
+ this.value = value;
+ this.metadata = metadata;
+ this.pos = pos;
+ // There is currently only one allowed version.
+ if (metadata.length < 1 || (metadata[0] & VERSION_MASK) != VERSION) {
+ throw malformedVariant();
+ }
+ }
+
+ public byte[] getValue() {
+ if (pos == 0) return value;
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ return Arrays.copyOfRange(value, pos, pos + size);
+ }
+
+ public byte[] getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * @return the boolean value
+ */
+ public boolean getBoolean() {
+ return VariantUtil.getBoolean(value, pos);
+ }
+
+ /**
+ * @return the long value
+ */
+ public long getLong() {
+ return VariantUtil.getLong(value, pos);
+ }
+
+ /**
+ * @return the double value
+ */
+ public double getDouble() {
+ return VariantUtil.getDouble(value, pos);
+ }
+
+ /**
+ * @return the decimal value
+ */
+ public BigDecimal getDecimal() {
+ return VariantUtil.getDecimal(value, pos);
+ }
+
+ /**
+ * @return the float value
+ */
+ public float getFloat() {
+ return VariantUtil.getFloat(value, pos);
+ }
+
+ /**
+ * @return the binary value
+ */
+ public byte[] getBinary() {
+ return VariantUtil.getBinary(value, pos);
+ }
+
+ /**
+ * @return the string value
+ */
+ public String getString() {
+ return VariantUtil.getString(value, pos);
+ }
+
+ /**
+ * @return the type info bits from a variant value
+ */
+ public int getTypeInfo() {
+ return VariantUtil.getTypeInfo(value, pos);
+ }
+
+ /**
+ * @return the type of the variant value
+ */
+ public Type getType() {
+ return VariantUtil.getType(value, pos);
+ }
+
+ /**
+ * @return the number of object fields in the variant. `getType()` must be `Type.OBJECT`.
+ */
+ public int objectSize() {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> size);
+ }
+
+ // Find the field value whose key is equal to `key`. Return null if the key is not found.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
+
+ /**
+ * Returns the object field Variant value whose key is equal to `key`.
+ * Return null if the key is not found. `getType()` must be `Type.OBJECT`.
+ * @param key the key to look up
+ * @return the field value whose key is equal to `key`, or null if key is not found
+ */
+ public Variant getFieldByKey(String key) {
+ return handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ // Use linear search for a short list. Switch to binary search when the length reaches
+ // `BINARY_SEARCH_THRESHOLD`.
+ final int BINARY_SEARCH_THRESHOLD = 32;
+ if (size < BINARY_SEARCH_THRESHOLD) {
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ if (key.equals(getMetadataKey(metadata, id))) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ } else {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ // Use unsigned right shift to compute the middle of `low` and `high`. This is not only a
+ // performance optimization, because it can properly handle the case where `low + high`
+ // overflows int.
+ int mid = (low + high) >>> 1;
+ int id = readUnsigned(value, idStart + idSize * mid, idSize);
+ int cmp = getMetadataKey(metadata, id).compareTo(key);
+ if (cmp < 0) {
+ low = mid + 1;
+ } else if (cmp > 0) {
+ high = mid - 1;
+ } else {
+ int offset = readUnsigned(value, offsetStart + offsetSize * mid, offsetSize);
+ return new Variant(value, metadata, dataStart + offset);
+ }
+ }
+ }
+ return null;
+ });
+ }
+
+ /**
+ * A field in a Variant object.
+ */
+ public static final class ObjectField {
+ public final String key;
+ public final Variant value;
+
+ public ObjectField(String key, Variant value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ // Get the object field at the `index` slot. Return null if `index` is out of the bound of
+ // `[0, objectSize())`.
+ // It is only legal to call it when `getType()` is `Type.OBJECT`.
Review Comment:
removed.
##########
parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java:
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.exc.InputCoercionException;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+/**
+ * Builder for creating Variant value and metadata.
+ */
+public class VariantBuilder {
+ public VariantBuilder(boolean allowDuplicateKeys) {
+ this(allowDuplicateKeys, DEFAULT_SIZE_LIMIT);
+ }
+
+ public VariantBuilder(boolean allowDuplicateKeys, int sizeLimitBytes) {
+ this.allowDuplicateKeys = allowDuplicateKeys;
+ this.sizeLimitBytes = sizeLimitBytes;
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json) throws IOException {
+ return parseJson(json, new VariantBuilder(false));
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json, VariantBuilder builder) throws IOException {
+ try (JsonParser parser = new JsonFactory().createParser(json)) {
+ parser.nextToken();
+ return parseJson(parser, builder);
+ }
+ }
+
+ /**
+ * Parse a JSON parser as a Variant value.
+ * @param parser the JSON parser to use
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(JsonParser parser, VariantBuilder builder) throws IOException {
+ builder.buildFromJsonParser(parser);
+ return builder.result();
+ }
+
+ /**
+ * @return the Variant value
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public Variant result() {
+ int numKeys = dictionaryKeys.size();
+ // Use long to avoid overflow in accumulating lengths.
+ long dictionaryStringSize = 0;
+ for (byte[] key : dictionaryKeys) {
+ dictionaryStringSize += key.length;
+ }
+ // Determine the number of bytes required per offset entry.
+ // The largest offset is the one-past-the-end value, which is total string size. It's very
+ // unlikely that the number of keys could be larger, but incorporate that into the calculation
+ // in case of pathological data.
+ long maxSize = Math.max(dictionaryStringSize, numKeys);
+ if (maxSize > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
Review Comment:
Updated with more details in the message. Example: `Variant size exceeds the limit of 100 bytes. Estimated size: 256 bytes`
##########
parquet-variant/src/main/java/org/apache/parquet/variant/VariantBuilder.java:
##########
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.parquet.variant;
+
+import static org.apache.parquet.variant.VariantUtil.*;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.exc.InputCoercionException;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+/**
+ * Builder for creating Variant value and metadata.
+ */
+public class VariantBuilder {
+ public VariantBuilder(boolean allowDuplicateKeys) {
+ this(allowDuplicateKeys, DEFAULT_SIZE_LIMIT);
+ }
+
+ public VariantBuilder(boolean allowDuplicateKeys, int sizeLimitBytes) {
+ this.allowDuplicateKeys = allowDuplicateKeys;
+ this.sizeLimitBytes = sizeLimitBytes;
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json) throws IOException {
+ return parseJson(json, new VariantBuilder(false));
+ }
+
+ /**
+ * Parse a JSON string as a Variant value.
+ * @param json the JSON string to parse
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(String json, VariantBuilder builder) throws IOException {
+ try (JsonParser parser = new JsonFactory().createParser(json)) {
+ parser.nextToken();
+ return parseJson(parser, builder);
+ }
+ }
+
+ /**
+ * Parse a JSON parser as a Variant value.
+ * @param parser the JSON parser to use
+ * @param builder the VariantBuilder to use for building the Variant
+ * @return the Variant value
+ * @throws IOException if any JSON parsing error happens
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public static Variant parseJson(JsonParser parser, VariantBuilder builder) throws IOException {
+ builder.buildFromJsonParser(parser);
+ return builder.result();
+ }
+
+ /**
+ * @return the Variant value
+ * @throws VariantSizeLimitException if the resulting variant value or metadata would exceed
+ * the size limit
+ */
+ public Variant result() {
+ int numKeys = dictionaryKeys.size();
+ // Use long to avoid overflow in accumulating lengths.
+ long dictionaryStringSize = 0;
+ for (byte[] key : dictionaryKeys) {
+ dictionaryStringSize += key.length;
+ }
+ // Determine the number of bytes required per offset entry.
+ // The largest offset is the one-past-the-end value, which is total string size. It's very
+ // unlikely that the number of keys could be larger, but incorporate that into the calculation
+ // in case of pathological data.
+ long maxSize = Math.max(dictionaryStringSize, numKeys);
+ if (maxSize > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ int offsetSize = getMinIntegerSize((int) maxSize);
+
+ int offsetStart = 1 + offsetSize;
+ int stringStart = offsetStart + (numKeys + 1) * offsetSize;
+ long metadataSize = stringStart + dictionaryStringSize;
+
+ if (metadataSize > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ byte[] metadata = new byte[(int) metadataSize];
+ int headerByte = VERSION | ((offsetSize - 1) << 6);
+ writeLong(metadata, 0, headerByte, 1);
+ writeLong(metadata, 1, numKeys, offsetSize);
+ int currentOffset = 0;
+ for (int i = 0; i < numKeys; ++i) {
+ writeLong(metadata, offsetStart + i * offsetSize, currentOffset, offsetSize);
+ byte[] key = dictionaryKeys.get(i);
+ System.arraycopy(key, 0, metadata, stringStart + currentOffset, key.length);
+ currentOffset += key.length;
+ }
+ writeLong(metadata, offsetStart + numKeys * offsetSize, currentOffset, offsetSize);
+ return new Variant(Arrays.copyOfRange(writeBuffer, 0, writePos), metadata);
+ }
+
+ public void appendString(String str) {
+ byte[] text = str.getBytes(StandardCharsets.UTF_8);
+ boolean longStr = text.length > MAX_SHORT_STR_SIZE;
+ checkCapacity((longStr ? 1 + U32_SIZE : 1) + text.length);
+ if (longStr) {
+ writeBuffer[writePos++] = primitiveHeader(LONG_STR);
+ writeLong(writeBuffer, writePos, text.length, U32_SIZE);
+ writePos += U32_SIZE;
+ } else {
+ writeBuffer[writePos++] = shortStrHeader(text.length);
+ }
+ System.arraycopy(text, 0, writeBuffer, writePos, text.length);
+ writePos += text.length;
+ }
+
+ public void appendNull() {
+ checkCapacity(1);
+ writeBuffer[writePos++] = primitiveHeader(NULL);
+ }
+
+ public void appendBoolean(boolean b) {
+ checkCapacity(1);
+ writeBuffer[writePos++] = primitiveHeader(b ? TRUE : FALSE);
+ }
+
+ /**
+ * Appends a long value to the variant builder. The actual encoded integer type depends on the
+ * value range of the long value.
+ * @param l the long value to append
+ */
+ public void appendLong(long l) {
+ checkCapacity(1 + 8);
+ if (l == (byte) l) {
+ writeBuffer[writePos++] = primitiveHeader(INT1);
+ writeLong(writeBuffer, writePos, l, 1);
+ writePos += 1;
+ } else if (l == (short) l) {
+ writeBuffer[writePos++] = primitiveHeader(INT2);
+ writeLong(writeBuffer, writePos, l, 2);
+ writePos += 2;
+ } else if (l == (int) l) {
+ writeBuffer[writePos++] = primitiveHeader(INT4);
+ writeLong(writeBuffer, writePos, l, 4);
+ writePos += 4;
+ } else {
+ writeBuffer[writePos++] = primitiveHeader(INT8);
+ writeLong(writeBuffer, writePos, l, 8);
+ writePos += 8;
+ }
+ }
+
+ public void appendDouble(double d) {
+ checkCapacity(1 + 8);
+ writeBuffer[writePos++] = primitiveHeader(DOUBLE);
+ writeLong(writeBuffer, writePos, Double.doubleToLongBits(d), 8);
+ writePos += 8;
+ }
+
+ /**
+ * Appends a decimal value to the variant builder. The actual encoded decimal type depends on the
+ * precision and scale of the decimal value.
+ * @param d the decimal value to append
+ */
+ public void appendDecimal(BigDecimal d) {
+ checkCapacity(2 + 16);
+ BigInteger unscaled = d.unscaledValue();
+ if (d.scale() <= MAX_DECIMAL4_PRECISION && d.precision() <= MAX_DECIMAL4_PRECISION) {
+ writeBuffer[writePos++] = primitiveHeader(DECIMAL4);
+ writeBuffer[writePos++] = (byte) d.scale();
+ writeLong(writeBuffer, writePos, unscaled.intValueExact(), 4);
+ writePos += 4;
+ } else if (d.scale() <= MAX_DECIMAL8_PRECISION && d.precision() <= MAX_DECIMAL8_PRECISION) {
+ writeBuffer[writePos++] = primitiveHeader(DECIMAL8);
+ writeBuffer[writePos++] = (byte) d.scale();
+ writeLong(writeBuffer, writePos, unscaled.longValueExact(), 8);
+ writePos += 8;
+ } else {
+ assert d.scale() <= MAX_DECIMAL16_PRECISION && d.precision() <= MAX_DECIMAL16_PRECISION;
+ writeBuffer[writePos++] = primitiveHeader(DECIMAL16);
+ writeBuffer[writePos++] = (byte) d.scale();
+ // `toByteArray` returns a big-endian representation. We need to copy it reversely and sign
+ // extend it to 16 bytes.
+ byte[] bytes = unscaled.toByteArray();
+ for (int i = 0; i < bytes.length; ++i) {
+ writeBuffer[writePos + i] = bytes[bytes.length - 1 - i];
+ }
+ byte sign = (byte) (bytes[0] < 0 ? -1 : 0);
+ for (int i = bytes.length; i < 16; ++i) {
+ writeBuffer[writePos + i] = sign;
+ }
+ writePos += 16;
+ }
+ }
+
+ public void appendDate(int daysSinceEpoch) {
+ checkCapacity(1 + 4);
+ writeBuffer[writePos++] = primitiveHeader(DATE);
+ writeLong(writeBuffer, writePos, daysSinceEpoch, 4);
+ writePos += 4;
+ }
+
+ public void appendTimestamp(long microsSinceEpoch) {
+ checkCapacity(1 + 8);
+ writeBuffer[writePos++] = primitiveHeader(TIMESTAMP);
+ writeLong(writeBuffer, writePos, microsSinceEpoch, 8);
+ writePos += 8;
+ }
+
+ public void appendTimestampNtz(long microsSinceEpoch) {
+ checkCapacity(1 + 8);
+ writeBuffer[writePos++] = primitiveHeader(TIMESTAMP_NTZ);
+ writeLong(writeBuffer, writePos, microsSinceEpoch, 8);
+ writePos += 8;
+ }
+
+ public void appendFloat(float f) {
+ checkCapacity(1 + 4);
+ writeBuffer[writePos++] = primitiveHeader(FLOAT);
+ writeLong(writeBuffer, writePos, Float.floatToIntBits(f), 8);
+ writePos += 4;
+ }
+
+ public void appendBinary(byte[] binary) {
+ checkCapacity(1 + U32_SIZE + binary.length);
+ writeBuffer[writePos++] = primitiveHeader(BINARY);
+ writeLong(writeBuffer, writePos, binary.length, U32_SIZE);
+ writePos += U32_SIZE;
+ System.arraycopy(binary, 0, writeBuffer, writePos, binary.length);
+ writePos += binary.length;
+ }
+
+ /**
+ * Adds a key to the Variant dictionary. If the key already exists, the dictionary is unmodified.
+ * @param key the key to add
+ * @return the id of the key
+ */
+ public int addKey(String key) {
+ int id;
+ if (dictionary.containsKey(key)) {
+ id = dictionary.get(key);
+ } else {
+ id = dictionaryKeys.size();
+ dictionary.put(key, id);
+ dictionaryKeys.add(key.getBytes(StandardCharsets.UTF_8));
+ }
+ return id;
+ }
+
+ /**
+ * @return the current write position of the variant builder
+ */
+ public int getWritePos() {
+ return writePos;
+ }
+
+ // Finish writing a variant object after all of its fields have already been written. The process
+ // is as follows:
+ // 1. The caller calls `getWritePos` before writing any fields to obtain the `start` parameter.
+ // 2. The caller appends all the object fields to the builder. In the meantime, it should maintain
+ // the `fields` parameter. Before appending each field, it should append an entry to `fields` to
+ // record the offset of the field. The offset is computed as `getWritePos() - start`.
+ // 3. The caller calls `finishWritingObject` to finish writing a variant object.
+ //
+ // This function is responsible to sort the fields by key. If there are duplicate field keys:
+ // - when `allowDuplicateKeys` is true, the field with the greatest offset value (the last
+ // appended one) is kept.
+ // - otherwise, throw an exception.
+ /**
+ * Finish writing a Variant object after all of its fields have already been written. The process
+ * is as follows:
+ * 1. The caller calls `getWritePos()` before writing any fields to obtain the `start` parameter.
+ * 2. The caller appends all the object fields to the builder. In the meantime, it should maintain
+ * the `fields` parameter. Before appending each field, it should append an entry to `fields` to
+ * record the offset of the field. The offset is computed as `getWritePos() - start`.
+ * 3. The caller calls `finishWritingObject` to finish writing the Variant object.
+ *
+ * This method will sort the fields by key. If there are duplicate field keys:
+ * - when `allowDuplicateKeys` is true, the field with the greatest offset value (the last
+ * appended one) is kept.
+ * - otherwise, throw an exception.
+ * @param start the start position of the object in the write buffer
+ * @param fields the list of `FieldEntry` in the object
+ * @throws VariantDuplicateKeyException if there are duplicate keys and `allowDuplicateKeys` is
+ * false
+ */
+ public void finishWritingObject(int start, ArrayList<FieldEntry> fields) {
+ int size = fields.size();
+ Collections.sort(fields);
+ int maxId = size == 0 ? 0 : fields.get(0).id;
+ if (allowDuplicateKeys) {
+ int distinctPos = 0;
+ // Maintain a list of distinct keys in-place.
+ for (int i = 1; i < size; ++i) {
+ maxId = Math.max(maxId, fields.get(i).id);
+ if (fields.get(i).id == fields.get(i - 1).id) {
+ // Found a duplicate key. Keep the field with the greater offset.
+ if (fields.get(distinctPos).offset < fields.get(i).offset) {
+ fields.set(distinctPos, fields.get(distinctPos).withNewOffset(fields.get(i).offset));
+ }
+ } else {
+ // Found a distinct key. Add the field to the list.
+ ++distinctPos;
+ fields.set(distinctPos, fields.get(i));
+ }
+ }
+ if (distinctPos + 1 < fields.size()) {
+ size = distinctPos + 1;
+ // Resize `fields` to `size`.
+ fields.subList(size, fields.size()).clear();
+ // Sort the fields by offsets so that we can move the value data of each field to the new
+ // offset without overwriting the fields after it.
+ fields.sort(Comparator.comparingInt(f -> f.offset));
+ int currentOffset = 0;
+ for (int i = 0; i < size; ++i) {
+ int oldOffset = fields.get(i).offset;
+ int fieldSize = VariantUtil.valueSize(writeBuffer, start + oldOffset);
+ System.arraycopy(writeBuffer, start + oldOffset, writeBuffer, start + currentOffset, fieldSize);
+ fields.set(i, fields.get(i).withNewOffset(currentOffset));
+ currentOffset += fieldSize;
+ }
+ writePos = start + currentOffset;
+ // Change back to the sort order by field keys, required by the Variant specification.
+ Collections.sort(fields);
+ }
+ } else {
+ for (int i = 1; i < size; ++i) {
+ maxId = Math.max(maxId, fields.get(i).id);
+ String key = fields.get(i).key;
+ if (key.equals(fields.get(i - 1).key)) {
+ throw new VariantDuplicateKeyException(key);
+ }
+ }
+ }
+ int dataSize = writePos - start;
+ boolean largeSize = size > U8_MAX;
+ int sizeBytes = largeSize ? U32_SIZE : 1;
+ int idSize = getMinIntegerSize(maxId);
+ int offsetSize = getMinIntegerSize(dataSize);
+ // The space for header byte, object size, id list, and offset list.
+ int headerSize = 1 + sizeBytes + size * idSize + (size + 1) * offsetSize;
+ checkCapacity(headerSize);
+ // Shift the just-written field data to make room for the object header section.
+ System.arraycopy(writeBuffer, start, writeBuffer, start + headerSize, dataSize);
+ writePos += headerSize;
+ writeBuffer[start] = objectHeader(largeSize, idSize, offsetSize);
+ writeLong(writeBuffer, start + 1, size, sizeBytes);
+ int idStart = start + 1 + sizeBytes;
+ int offsetStart = idStart + size * idSize;
+ for (int i = 0; i < size; ++i) {
+ writeLong(writeBuffer, idStart + i * idSize, fields.get(i).id, idSize);
+ writeLong(writeBuffer, offsetStart + i * offsetSize, fields.get(i).offset, offsetSize);
+ }
+ writeLong(writeBuffer, offsetStart + size * offsetSize, dataSize, offsetSize);
+ }
+
+ /**
+ * Finish writing a Variant array after all of its elements have already been written. The process
+ * is similar to that of `finishWritingObject`.
+ * @param start the start position of the array in the write buffer
+ * @param offsets the list of offsets of the array elements
+ */
+ public void finishWritingArray(int start, ArrayList<Integer> offsets) {
+ int dataSize = writePos - start;
+ int size = offsets.size();
+ boolean largeSize = size > U8_MAX;
+ int sizeBytes = largeSize ? U32_SIZE : 1;
+ int offsetSize = getMinIntegerSize(dataSize);
+ // The space for header byte, object size, and offset list.
+ int headerSize = 1 + sizeBytes + (size + 1) * offsetSize;
+ checkCapacity(headerSize);
+ // Shift the just-written field data to make room for the header section.
+ System.arraycopy(writeBuffer, start, writeBuffer, start + headerSize, dataSize);
+ writePos += headerSize;
+ writeBuffer[start] = arrayHeader(largeSize, offsetSize);
+ writeLong(writeBuffer, start + 1, size, sizeBytes);
+ int offsetStart = start + 1 + sizeBytes;
+ for (int i = 0; i < size; ++i) {
+ writeLong(writeBuffer, offsetStart + i * offsetSize, offsets.get(i), offsetSize);
+ }
+ writeLong(writeBuffer, offsetStart + size * offsetSize, dataSize, offsetSize);
+ }
+
+ /**
+ * Appends a Variant value to the Variant builder. The input Variant keys must be inserted into
+ * the builder dictionary and rebuilt with new field ids. For scalar values in the input
+ * Variant, we can directly copy the binary slice.
+ * @param v the Variant value to append
+ */
+ public void appendVariant(Variant v) {
+ appendVariantImpl(v.value, v.metadata, v.pos);
+ }
+
+ private void appendVariantImpl(byte[] value, byte[] metadata, int pos) {
+ checkIndex(pos, value.length);
+ int basicType = value[pos] & BASIC_TYPE_MASK;
+ switch (basicType) {
+ case OBJECT:
+ handleObject(value, pos, (size, idSize, offsetSize, idStart, offsetStart, dataStart) -> {
+ ArrayList<FieldEntry> fields = new ArrayList<>(size);
+ int start = writePos;
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ int elementPos = dataStart + offset;
+ String key = getMetadataKey(metadata, id);
+ int newId = addKey(key);
+ fields.add(new FieldEntry(key, newId, writePos - start));
+ appendVariantImpl(value, metadata, elementPos);
+ }
+ finishWritingObject(start, fields);
+ return null;
+ });
+ break;
+ case ARRAY:
+ handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ ArrayList<Integer> offsets = new ArrayList<>(size);
+ int start = writePos;
+ for (int i = 0; i < size; ++i) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i, offsetSize);
+ int elementPos = dataStart + offset;
+ offsets.add(writePos - start);
+ appendVariantImpl(value, metadata, elementPos);
+ }
+ finishWritingArray(start, offsets);
+ return null;
+ });
+ break;
+ default:
+ shallowAppendVariantImpl(value, pos);
+ break;
+ }
+ }
+
+ private void shallowAppendVariantImpl(byte[] value, int pos) {
+ int size = valueSize(value, pos);
+ checkIndex(pos + size - 1, value.length);
+ checkCapacity(size);
+ System.arraycopy(value, pos, writeBuffer, writePos, size);
+ writePos += size;
+ }
+
+ private void checkCapacity(int additionalBytes) {
+ int requiredBytes = writePos + additionalBytes;
+ if (requiredBytes > writeBuffer.length) {
+ // Allocate a new buffer with a capacity of the next power of 2 of `requiredBytes`.
+ int newCapacity = Integer.highestOneBit(requiredBytes);
+ newCapacity = newCapacity < requiredBytes ? newCapacity * 2 : newCapacity;
+ if (newCapacity > sizeLimitBytes) {
+ throw new VariantSizeLimitException();
+ }
+ byte[] newValue = new byte[newCapacity];
+ System.arraycopy(writeBuffer, 0, newValue, 0, writePos);
+ writeBuffer = newValue;
+ }
+ }
+
+ // Temporarily store the information of a field. We need to collect all fields in an JSON object,
+ // sort them by their keys, and build the variant object in sorted order.
+
+ /**
+ * Class to store the information of a Variant object field. We need to collect all fields of
+ * an object, sort them by their keys, and build the Variant object in sorted order.
+ */
+ public static final class FieldEntry implements Comparable<FieldEntry> {
+ final String key;
+ final int id;
+ final int offset;
+
+ public FieldEntry(String key, int id, int offset) {
+ this.key = key;
+ this.id = id;
+ this.offset = offset;
+ }
+
+ FieldEntry withNewOffset(int newOffset) {
+ return new FieldEntry(key, id, newOffset);
+ }
+
+ @Override
+ public int compareTo(FieldEntry other) {
+ return key.compareTo(other.key);
+ }
+ }
+
+ private void buildFromJsonParser(JsonParser parser) throws IOException {
+ JsonToken token = parser.currentToken();
+ if (token == null) {
+ throw new JsonParseException(parser, "Unexpected null token");
+ }
+ switch (token) {
+ case START_OBJECT: {
+ ArrayList<FieldEntry> fields = new ArrayList<>();
+ int start = writePos;
+ while (parser.nextToken() != JsonToken.END_OBJECT) {
+ String key = parser.currentName();
+ parser.nextToken();
+ int id = addKey(key);
+ fields.add(new FieldEntry(key, id, writePos - start));
+ buildFromJsonParser(parser);
+ }
+ finishWritingObject(start, fields);
+ break;
+ }
+ case START_ARRAY: {
+ ArrayList<Integer> offsets = new ArrayList<>();
+ int start = writePos;
+ while (parser.nextToken() != JsonToken.END_ARRAY) {
+ offsets.add(writePos - start);
+ buildFromJsonParser(parser);
+ }
+ finishWritingArray(start, offsets);
+ break;
+ }
+ case VALUE_STRING:
+ appendString(parser.getText());
+ break;
+ case VALUE_NUMBER_INT:
+ try {
+ appendLong(parser.getLongValue());
+ } catch (InputCoercionException ignored) {
+ // If the value doesn't fit any integer type, parse it as decimal or floating instead.
+ parseAndAppendFloatingPoint(parser);
Review Comment:
Yeah, this is a tricky situation. We decided to allow parsing this type of valid JSON and not return an error, since the JSON is technically valid. It is not ideal that a valid JSON string hits an error. This behavior is similar to how [Snowflake's variant parses JSON](https://docs.snowflake.com/en/sql-reference/functions/parse_json#usage-notes).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@parquet.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@parquet.apache.org
For additional commands, e-mail: issues-help@parquet.apache.org