You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ga...@apache.org on 2026/06/25 08:18:03 UTC
(parquet-java) branch master updated: PARQUET-2249: Add IEEE-754 total order and nan count for floating types (#3393)
This is an automated email from the ASF dual-hosted git repository.
wgtmac pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-java.git
The following commit(s) were added to refs/heads/master by this push:
new 071a84122 PARQUET-2249: Add IEEE-754 total order and nan count for floating types (#3393)
071a84122 is described below
commit 071a84122f8c147554e56f44b073402e8a6347cc
Author: Gang Wu <us...@gmail.com>
AuthorDate: Thu Jun 25 16:17:58 2026 +0800
PARQUET-2249: Add IEEE-754 total order and nan count for floating types (#3393)
### Rationale for this change
This implements the parquet-format IEEE 754 total order column-order work from apache/parquet-format#514 in parquet-java. The existing type-defined order remains the default.
While adding the new order, this also fixes floating-point NaN handling so parquet-java preserves the exact FLOAT, DOUBLE, and FLOAT16 bit patterns supplied by applications, including NaN sign and payload bits. Filters are updated to avoid false negatives when NaN semantics cannot be answered safely from metadata.
### What changes are included in this PR?
- Add IEEE_754_TOTAL_ORDER support for FLOAT, DOUBLE, and FLOAT16, including comparators that distinguish -0/+0 and NaN bit patterns.
- Add floating-point nan_count support in statistics and column indexes, with IEEE-754-specific statistics implementations.
- Preserve raw floating-point bit patterns in parquet-java floating encodings, including dictionary and byte-stream-split paths.
- Update statistics, column-index, dictionary, bloom-filter, and record-level filtering so NaN values are handled conservatively where required.
- Add coverage for statistics, column indexes, dictionary filtering, bloom filtering, record-level filtering, Float16, and parquet-testing interop files.
### Are these changes tested?
Yes. This PR adds focused unit and end-to-end tests for the new column order, NaN counts, NaN filtering behavior, raw-bit preservation, and interop with parquet-testing files.
### Are there any user-facing changes?
Yes. Users can opt into IEEE_754_TOTAL_ORDER for floating columns. The default column order is unchanged, and existing files remain readable. Filtering around NaN values may be more conservative to avoid dropping data that can still match.
Closes #406
---
.../column/statistics/BinaryStatistics.java | 16 +-
.../column/statistics/DoubleStatistics.java | 41 +-
.../column/statistics/Float16Statistics.java | 92 ++++
.../parquet/column/statistics/FloatStatistics.java | 41 +-
.../column/statistics/IEEE754DoubleStatistics.java | 72 +++
.../statistics/IEEE754Float16Statistics.java | 77 +++
.../column/statistics/IEEE754FloatStatistics.java | 72 +++
.../parquet/column/statistics/Statistics.java | 162 ++++--
.../ByteStreamSplitValuesWriter.java | 4 +-
.../values/dictionary/DictionaryValuesWriter.java | 56 +--
.../columnindex/BinaryColumnIndexBuilder.java | 62 ++-
.../internal/column/columnindex/ColumnIndex.java | 8 +
.../column/columnindex/ColumnIndexBuilder.java | 260 +++++++++-
.../columnindex/DoubleColumnIndexBuilder.java | 60 ++-
.../columnindex/FloatColumnIndexBuilder.java | 60 ++-
.../org/apache/parquet/schema/ColumnOrder.java | 15 +-
.../parquet/schema/LogicalTypeAnnotation.java | 8 +
.../apache/parquet/schema/PrimitiveComparator.java | 61 +++
.../org/apache/parquet/schema/PrimitiveType.java | 66 ++-
.../column/statistics/TestStatisticsNanCount.java | 374 ++++++++++++++
.../ByteStreamSplitValuesEndToEndTest.java | 64 +++
.../column/values/dictionary/TestDictionary.java | 62 +++
.../columnindex/TestColumnIndexBuilderNaN.java | 423 ++++++++++++++++
.../filter2/columnindex/TestColumnIndexFilter.java | 31 +-
.../org/apache/parquet/schema/TestMessageType.java | 4 +-
.../parquet/schema/TestPrimitiveComparator.java | 119 ++++-
.../bytes/LittleEndianDataOutputStream.java | 10 +-
.../filter2/bloomfilterlevel/BloomFilterImpl.java | 41 ++
.../filter2/dictionarylevel/DictionaryFilter.java | 136 +++--
.../filter2/statisticslevel/StatisticsFilter.java | 197 +++++++-
.../format/converter/ParquetMetadataConverter.java | 57 ++-
.../parquet/hadoop/rewrite/ParquetRewriter.java | 4 +
.../bloomfilterlevel/TestBloomFilterImpl.java | 131 +++++
.../dictionarylevel/DictionaryFilterTest.java | 211 +++++++-
.../statisticslevel/TestStatisticsFilter.java | 407 +++++++++++++++
.../converter/TestParquetMetadataConverter.java | 181 +++++++
.../FloatingPointNanInteropFileGenerator.java | 411 ++++++++++++++++
.../TestInterOpReadFloatingPointNanCount.java | 436 ++++++++++++++++
.../parquet/statistics/TestFloat16Statistics.java | 8 +-
.../statistics/TestIeee754TotalOrderE2E.java | 546 +++++++++++++++++++++
40 files changed, 4857 insertions(+), 229 deletions(-)
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/statistics/BinaryStatistics.java b/parquet-column/src/main/java/org/apache/parquet/column/statistics/BinaryStatistics.java
index 9488a3849..b17110680 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/statistics/BinaryStatistics.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/statistics/BinaryStatistics.java
@@ -28,8 +28,8 @@ public class BinaryStatistics extends Statistics<Binary> {
private static final PrimitiveType DEFAULT_FAKE_TYPE =
Types.optional(PrimitiveType.PrimitiveTypeName.BINARY).named("fake_binary_type");
- private Binary max;
- private Binary min;
+ protected Binary max;
+ protected Binary min;
/**
* @deprecated will be removed in 2.0.0. Use {@link Statistics#createStats(org.apache.parquet.schema.Type)} instead
@@ -43,7 +43,7 @@ public class BinaryStatistics extends Statistics<Binary> {
super(type);
}
- private BinaryStatistics(BinaryStatistics other) {
+ protected BinaryStatistics(BinaryStatistics other) {
super(other.type());
if (other.hasNonNullValue()) {
initializeStats(other.min, other.max);
@@ -57,10 +57,12 @@ public class BinaryStatistics extends Statistics<Binary> {
min = value.copy();
max = value.copy();
this.markAsNotEmpty();
- } else if (comparator().compare(min, value) > 0) {
- min = value.copy();
- } else if (comparator().compare(max, value) < 0) {
- max = value.copy();
+ } else {
+ if (comparator().compare(min, value) > 0) {
+ min = value.copy();
+ } else if (comparator().compare(max, value) < 0) {
+ max = value.copy();
+ }
}
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/statistics/DoubleStatistics.java b/parquet-column/src/main/java/org/apache/parquet/column/statistics/DoubleStatistics.java
index 3fe8a3553..d7e0f6ca2 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/statistics/DoubleStatistics.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/statistics/DoubleStatistics.java
@@ -28,8 +28,8 @@ public class DoubleStatistics extends Statistics<Double> {
private static final PrimitiveType DEFAULT_FAKE_TYPE =
Types.optional(PrimitiveType.PrimitiveTypeName.DOUBLE).named("fake_double_type");
- private double max;
- private double min;
+ protected double max;
+ protected double min;
/**
* @deprecated will be removed in 2.0.0. Use {@link Statistics#createStats(org.apache.parquet.schema.Type)} instead
@@ -41,22 +41,28 @@ public class DoubleStatistics extends Statistics<Double> {
DoubleStatistics(PrimitiveType type) {
super(type);
+ incrementNanCount(0);
}
- private DoubleStatistics(DoubleStatistics other) {
+ protected DoubleStatistics(DoubleStatistics other) {
super(other.type());
if (other.hasNonNullValue()) {
initializeStats(other.min, other.max);
}
setNumNulls(other.getNumNulls());
+ incrementNanCount(other.getNanCount());
}
@Override
public void updateStats(double value) {
+ if (Double.isNaN(value)) {
+ incrementNanCount();
+ return;
+ }
if (!this.hasNonNullValue()) {
- initializeStats(value, value);
+ initializeStats(normalizeMinValue(value), normalizeMaxValue(value));
} else {
- updateStats(value, value);
+ updateStats(normalizeMinValue(value), normalizeMaxValue(value));
}
}
@@ -79,12 +85,12 @@ public class DoubleStatistics extends Statistics<Double> {
@Override
public byte[] getMaxBytes() {
- return BytesUtils.longToBytes(Double.doubleToLongBits(max));
+ return BytesUtils.longToBytes(Double.doubleToRawLongBits(max));
}
@Override
public byte[] getMinBytes() {
- return BytesUtils.longToBytes(Double.doubleToLongBits(min));
+ return BytesUtils.longToBytes(Double.doubleToRawLongBits(min));
}
@Override
@@ -98,6 +104,19 @@ public class DoubleStatistics extends Statistics<Double> {
}
public void updateStats(double min_value, double max_value) {
+ if (Double.isNaN(min_value) && Double.isNaN(max_value)) {
+ return;
+ }
+ if (Double.isNaN(min_value)) {
+ min_value = max_value;
+ }
+ if (Double.isNaN(max_value)) {
+ max_value = min_value;
+ }
+
+ min_value = normalizeMinValue(min_value);
+ max_value = normalizeMaxValue(max_value);
+
if (comparator().compare(min, min_value) > 0) {
min = min_value;
}
@@ -144,6 +163,14 @@ public class DoubleStatistics extends Statistics<Double> {
this.markAsNotEmpty();
}
+ private static double normalizeMinValue(double value) {
+ return value == 0.0 ? -0.0 : value;
+ }
+
+ private static double normalizeMaxValue(double value) {
+ return value == 0.0 ? 0.0 : value;
+ }
+
@Override
public DoubleStatistics copy() {
return new DoubleStatistics(this);
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/statistics/Float16Statistics.java b/parquet-column/src/main/java/org/apache/parquet/column/statistics/Float16Statistics.java
new file mode 100644
index 000000000..1d7e69c28
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/statistics/Float16Statistics.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.statistics;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.Float16;
+import org.apache.parquet.schema.PrimitiveType;
+
+public class Float16Statistics extends BinaryStatistics {
+
+ Float16Statistics(PrimitiveType type) {
+ super(type);
+ incrementNanCount(0);
+ }
+
+ private Float16Statistics(Float16Statistics other) {
+ super(other);
+ incrementNanCount(other.getNanCount());
+ }
+
+ @Override
+ public void updateStats(Binary value) {
+ if (Float16.isNaN(value.get2BytesLittleEndian())) {
+ incrementNanCount();
+ return;
+ }
+ if (!this.hasNonNullValue()) {
+ initializeStats(normalizeMinValue(value), normalizeMaxValue(value));
+ } else {
+ updateStats(normalizeMinValue(value), normalizeMaxValue(value));
+ }
+ }
+
+ @Override
+ @Deprecated
+ public void updateStats(Binary min_value, Binary max_value) {
+ boolean minIsNaN = Float16.isNaN(min_value.get2BytesLittleEndian());
+ boolean maxIsNaN = Float16.isNaN(max_value.get2BytesLittleEndian());
+ if (minIsNaN && maxIsNaN) {
+ return;
+ }
+ if (minIsNaN) {
+ min_value = max_value;
+ }
+ if (maxIsNaN) {
+ max_value = min_value;
+ }
+
+ min_value = normalizeMinValue(min_value);
+ max_value = normalizeMaxValue(max_value);
+
+ if (comparator().compare(min, min_value) > 0) {
+ min = min_value.copy();
+ }
+ if (comparator().compare(max, max_value) < 0) {
+ max = max_value.copy();
+ }
+ }
+
+ @Override
+ public Float16Statistics copy() {
+ return new Float16Statistics(this);
+ }
+
+ private Binary normalizeMinValue(Binary value) {
+ return isZero(value) ? Float16.NEGATIVE_ZERO_LITTLE_ENDIAN : value;
+ }
+
+ private Binary normalizeMaxValue(Binary value) {
+ return isZero(value) ? Float16.POSITIVE_ZERO_LITTLE_ENDIAN : value;
+ }
+
+ private static boolean isZero(Binary value) {
+ return (value.get2BytesLittleEndian() & 0x7fff) == 0;
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/statistics/FloatStatistics.java b/parquet-column/src/main/java/org/apache/parquet/column/statistics/FloatStatistics.java
index 5b743b688..562ec9175 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/statistics/FloatStatistics.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/statistics/FloatStatistics.java
@@ -28,8 +28,8 @@ public class FloatStatistics extends Statistics<Float> {
private static final PrimitiveType DEFAULT_FAKE_TYPE =
Types.optional(PrimitiveType.PrimitiveTypeName.FLOAT).named("fake_float_type");
- private float max;
- private float min;
+ protected float max;
+ protected float min;
/**
* @deprecated will be removed in 2.0.0. Use {@link Statistics#createStats(org.apache.parquet.schema.Type)} instead
@@ -42,22 +42,28 @@ public class FloatStatistics extends Statistics<Float> {
FloatStatistics(PrimitiveType type) {
super(type);
+ incrementNanCount(0);
}
- private FloatStatistics(FloatStatistics other) {
+ protected FloatStatistics(FloatStatistics other) {
super(other.type());
if (other.hasNonNullValue()) {
initializeStats(other.min, other.max);
}
setNumNulls(other.getNumNulls());
+ incrementNanCount(other.getNanCount());
}
@Override
public void updateStats(float value) {
+ if (Float.isNaN(value)) {
+ incrementNanCount();
+ return;
+ }
if (!this.hasNonNullValue()) {
- initializeStats(value, value);
+ initializeStats(normalizeMinValue(value), normalizeMaxValue(value));
} else {
- updateStats(value, value);
+ updateStats(normalizeMinValue(value), normalizeMaxValue(value));
}
}
@@ -80,12 +86,12 @@ public class FloatStatistics extends Statistics<Float> {
@Override
public byte[] getMaxBytes() {
- return BytesUtils.intToBytes(Float.floatToIntBits(max));
+ return BytesUtils.intToBytes(Float.floatToRawIntBits(max));
}
@Override
public byte[] getMinBytes() {
- return BytesUtils.intToBytes(Float.floatToIntBits(min));
+ return BytesUtils.intToBytes(Float.floatToRawIntBits(min));
}
@Override
@@ -99,6 +105,19 @@ public class FloatStatistics extends Statistics<Float> {
}
public void updateStats(float min_value, float max_value) {
+ if (Float.isNaN(min_value) && Float.isNaN(max_value)) {
+ return;
+ }
+ if (Float.isNaN(min_value)) {
+ min_value = max_value;
+ }
+ if (Float.isNaN(max_value)) {
+ max_value = min_value;
+ }
+
+ min_value = normalizeMinValue(min_value);
+ max_value = normalizeMaxValue(max_value);
+
if (comparator().compare(min, min_value) > 0) {
min = min_value;
}
@@ -145,6 +164,14 @@ public class FloatStatistics extends Statistics<Float> {
this.markAsNotEmpty();
}
+ private static float normalizeMinValue(float value) {
+ return value == 0.0f ? -0.0f : value;
+ }
+
+ private static float normalizeMaxValue(float value) {
+ return value == 0.0f ? 0.0f : value;
+ }
+
@Override
public FloatStatistics copy() {
return new FloatStatistics(this);
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/statistics/IEEE754DoubleStatistics.java b/parquet-column/src/main/java/org/apache/parquet/column/statistics/IEEE754DoubleStatistics.java
new file mode 100644
index 000000000..e5dba12bb
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/statistics/IEEE754DoubleStatistics.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.statistics;
+
+import org.apache.parquet.schema.PrimitiveType;
+
+public class IEEE754DoubleStatistics extends DoubleStatistics {
+
+ IEEE754DoubleStatistics(PrimitiveType type) {
+ super(type);
+ }
+
+ private IEEE754DoubleStatistics(IEEE754DoubleStatistics other) {
+ super(other);
+ }
+
+ @Override
+ public void updateStats(double value) {
+ if (Double.isNaN(value)) {
+ incrementNanCount();
+ }
+ if (!this.hasNonNullValue()) {
+ initializeStats(value, value);
+ } else {
+ updateStats(value, value);
+ }
+ }
+
+ @Override
+ public void updateStats(double min_value, double max_value) {
+ boolean minValueIsNaN = Double.isNaN(min_value);
+ boolean minIsNaN = Double.isNaN(min);
+ if (minValueIsNaN) {
+ if (minIsNaN && comparator().compare(min, min_value) > 0) {
+ min = min_value;
+ }
+ } else if (minIsNaN || comparator().compare(min, min_value) > 0) {
+ min = min_value;
+ }
+
+ boolean maxValueIsNaN = Double.isNaN(max_value);
+ boolean maxIsNaN = Double.isNaN(max);
+ if (maxValueIsNaN) {
+ if (maxIsNaN && comparator().compare(max, max_value) < 0) {
+ max = max_value;
+ }
+ } else if (maxIsNaN || comparator().compare(max, max_value) < 0) {
+ max = max_value;
+ }
+ }
+
+ @Override
+ public IEEE754DoubleStatistics copy() {
+ return new IEEE754DoubleStatistics(this);
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/statistics/IEEE754Float16Statistics.java b/parquet-column/src/main/java/org/apache/parquet/column/statistics/IEEE754Float16Statistics.java
new file mode 100644
index 000000000..31671496e
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/statistics/IEEE754Float16Statistics.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.statistics;
+
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.Float16;
+import org.apache.parquet.schema.PrimitiveType;
+
+public class IEEE754Float16Statistics extends BinaryStatistics {
+
+ IEEE754Float16Statistics(PrimitiveType type) {
+ super(type);
+ incrementNanCount(0);
+ }
+
+ private IEEE754Float16Statistics(IEEE754Float16Statistics other) {
+ super(other);
+ incrementNanCount(other.getNanCount());
+ }
+
+ @Override
+ public void updateStats(Binary value) {
+ if (Float16.isNaN(value.get2BytesLittleEndian())) {
+ incrementNanCount();
+ }
+ if (!this.hasNonNullValue()) {
+ initializeStats(value, value);
+ } else {
+ updateStats(value, value);
+ }
+ }
+
+ @Override
+ @Deprecated
+ public void updateStats(Binary min_value, Binary max_value) {
+ boolean minValueIsNaN = Float16.isNaN(min_value.get2BytesLittleEndian());
+ boolean minIsNaN = Float16.isNaN(min.get2BytesLittleEndian());
+ if (minValueIsNaN) {
+ if (minIsNaN && comparator().compare(min, min_value) > 0) {
+ min = min_value.copy();
+ }
+ } else if (minIsNaN || comparator().compare(min, min_value) > 0) {
+ min = min_value.copy();
+ }
+
+ boolean maxValueIsNaN = Float16.isNaN(max_value.get2BytesLittleEndian());
+ boolean maxIsNaN = Float16.isNaN(max.get2BytesLittleEndian());
+ if (maxValueIsNaN) {
+ if (maxIsNaN && comparator().compare(max, max_value) < 0) {
+ max = max_value.copy();
+ }
+ } else if (maxIsNaN || comparator().compare(max, max_value) < 0) {
+ max = max_value.copy();
+ }
+ }
+
+ @Override
+ public IEEE754Float16Statistics copy() {
+ return new IEEE754Float16Statistics(this);
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/statistics/IEEE754FloatStatistics.java b/parquet-column/src/main/java/org/apache/parquet/column/statistics/IEEE754FloatStatistics.java
new file mode 100644
index 000000000..8d286386d
--- /dev/null
+++ b/parquet-column/src/main/java/org/apache/parquet/column/statistics/IEEE754FloatStatistics.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.statistics;
+
+import org.apache.parquet.schema.PrimitiveType;
+
+public class IEEE754FloatStatistics extends FloatStatistics {
+
+ IEEE754FloatStatistics(PrimitiveType type) {
+ super(type);
+ }
+
+ private IEEE754FloatStatistics(IEEE754FloatStatistics other) {
+ super(other);
+ }
+
+ @Override
+ public void updateStats(float value) {
+ if (Float.isNaN(value)) {
+ incrementNanCount();
+ }
+ if (!this.hasNonNullValue()) {
+ initializeStats(value, value);
+ } else {
+ updateStats(value, value);
+ }
+ }
+
+ @Override
+ public void updateStats(float min_value, float max_value) {
+ boolean minValueIsNaN = Float.isNaN(min_value);
+ boolean minIsNaN = Float.isNaN(min);
+ if (minValueIsNaN) {
+ if (minIsNaN && comparator().compare(min, min_value) > 0) {
+ min = min_value;
+ }
+ } else if (minIsNaN || comparator().compare(min, min_value) > 0) {
+ min = min_value;
+ }
+
+ boolean maxValueIsNaN = Float.isNaN(max_value);
+ boolean maxIsNaN = Float.isNaN(max);
+ if (maxValueIsNaN) {
+ if (maxIsNaN && comparator().compare(max, max_value) < 0) {
+ max = max_value;
+ }
+ } else if (maxIsNaN || comparator().compare(max, max_value) < 0) {
+ max = max_value;
+ }
+ }
+
+ @Override
+ public IEEE754FloatStatistics copy() {
+ return new IEEE754FloatStatistics(this);
+ }
+}
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/statistics/Statistics.java b/parquet-column/src/main/java/org/apache/parquet/column/statistics/Statistics.java
index bee987773..aa7a14bde 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/statistics/Statistics.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/statistics/Statistics.java
@@ -21,6 +21,7 @@ package org.apache.parquet.column.statistics;
import java.util.Arrays;
import org.apache.parquet.column.UnknownColumnTypeException;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
import org.apache.parquet.schema.Float16;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.PrimitiveComparator;
@@ -40,10 +41,11 @@ public abstract class Statistics<T extends Comparable<T>> {
* Builder class to build Statistics objects. Used to read the statistics from the Parquet file.
*/
public static class Builder {
- private final PrimitiveType type;
+ protected final PrimitiveType type;
private byte[] min;
private byte[] max;
private long numNulls = -1;
+ private long nanCount = -1;
private Builder(PrimitiveType type) {
this.type = type;
@@ -64,12 +66,18 @@ public abstract class Statistics<T extends Comparable<T>> {
return this;
}
+ public Builder withNanCount(long nanCount) {
+ this.nanCount = nanCount;
+ return this;
+ }
+
public Statistics<?> build() {
Statistics<?> stats = createStats(type);
if (min != null && max != null) {
stats.setMinMaxFromBytes(min, max);
}
stats.num_nulls = this.numNulls;
+ stats.nan_count = this.nanCount;
return stats;
}
}
@@ -87,19 +95,24 @@ public abstract class Statistics<T extends Comparable<T>> {
if (stats.hasNonNullValue()) {
Float min = stats.genericGetMin();
Float max = stats.genericGetMax();
- // Drop min/max values in case of NaN as the sorting order of values is undefined for this case
if (min.isNaN() || max.isNaN()) {
- stats.setMinMax(0.0f, 0.0f);
- ((Statistics<?>) stats).hasNonNullValue = false;
- } else {
- // Updating min to -0.0 and max to +0.0 to ensure that no 0.0 values would be skipped
- if (Float.compare(min, 0.0f) == 0) {
- min = -0.0f;
- stats.setMinMax(min, max);
+ if (!type.columnOrder().equals(ColumnOrder.ieee754TotalOrder())) {
+ // For TYPE_DEFINED_ORDER: drop min/max values as NaN ordering is undefined
+ stats.setMinMax(0.0f, 0.0f);
+ ((Statistics<?>) stats).hasNonNullValue = false;
}
- if (Float.compare(max, -0.0f) == 0) {
- max = 0.0f;
- stats.setMinMax(min, max);
+ } else {
+ // For TYPE_DEFINED_ORDER, updating min to -0.0 and max to +0.0 to ensure that no 0.0 values
+ // would be skipped. For IEEE_754_TOTAL_ORDER, -0 < +0 is well-defined so no expansion needed.
+ if (!type.columnOrder().equals(ColumnOrder.ieee754TotalOrder())) {
+ if (Float.compare(min, 0.0f) == 0) {
+ min = -0.0f;
+ stats.setMinMax(min, max);
+ }
+ if (Float.compare(max, -0.0f) == 0) {
+ max = 0.0f;
+ stats.setMinMax(min, max);
+ }
}
}
}
@@ -120,19 +133,24 @@ public abstract class Statistics<T extends Comparable<T>> {
if (stats.hasNonNullValue()) {
Double min = stats.genericGetMin();
Double max = stats.genericGetMax();
- // Drop min/max values in case of NaN as the sorting order of values is undefined for this case
if (min.isNaN() || max.isNaN()) {
- stats.setMinMax(0.0, 0.0);
- ((Statistics<?>) stats).hasNonNullValue = false;
- } else {
- // Updating min to -0.0 and max to +0.0 to ensure that no 0.0 values would be skipped
- if (Double.compare(min, 0.0) == 0) {
- min = -0.0;
- stats.setMinMax(min, max);
+ if (!type.columnOrder().equals(ColumnOrder.ieee754TotalOrder())) {
+ // For TYPE_DEFINED_ORDER: drop min/max values as NaN ordering is undefined
+ stats.setMinMax(0.0, 0.0);
+ ((Statistics<?>) stats).hasNonNullValue = false;
}
- if (Double.compare(max, -0.0) == 0) {
- max = 0.0;
- stats.setMinMax(min, max);
+ } else {
+ // For TYPE_DEFINED_ORDER, updating min to -0.0 and max to +0.0 to ensure that no 0.0 values
+ // would be skipped. For IEEE_754_TOTAL_ORDER, -0 < +0 is well-defined so no expansion needed.
+ if (!type.columnOrder().equals(ColumnOrder.ieee754TotalOrder())) {
+ if (Double.compare(min, 0.0) == 0) {
+ min = -0.0;
+ stats.setMinMax(min, max);
+ }
+ if (Double.compare(max, -0.0) == 0) {
+ max = 0.0;
+ stats.setMinMax(min, max);
+ }
}
}
}
@@ -156,19 +174,24 @@ public abstract class Statistics<T extends Comparable<T>> {
Binary bMax = stats.genericGetMax();
short min = bMin.get2BytesLittleEndian();
short max = bMax.get2BytesLittleEndian();
- // Drop min/max values in case of NaN as the sorting order of values is undefined for this case
if (Float16.isNaN(min) || Float16.isNaN(max)) {
- stats.setMinMax(Float16.POSITIVE_ZERO_LITTLE_ENDIAN, Float16.POSITIVE_ZERO_LITTLE_ENDIAN);
- ((Statistics<?>) stats).hasNonNullValue = false;
- } else {
- // Updating min to -0.0 and max to +0.0 to ensure that no 0.0 values would be skipped
- if (min == (short) 0x0000) {
- bMin = Float16.NEGATIVE_ZERO_LITTLE_ENDIAN;
- stats.setMinMax(bMin, bMax);
+ if (!type.columnOrder().equals(ColumnOrder.ieee754TotalOrder())) {
+ // For TYPE_DEFINED_ORDER: drop min/max values as NaN ordering is undefined
+ stats.setMinMax(Float16.POSITIVE_ZERO_LITTLE_ENDIAN, Float16.POSITIVE_ZERO_LITTLE_ENDIAN);
+ ((Statistics<?>) stats).hasNonNullValue = false;
}
- if (max == (short) 0x8000) {
- bMax = Float16.POSITIVE_ZERO_LITTLE_ENDIAN;
- stats.setMinMax(bMin, bMax);
+ } else {
+ // For TYPE_DEFINED_ORDER, updating min to -0.0 and max to +0.0 to ensure that no 0.0 values
+ // would be skipped. For IEEE_754_TOTAL_ORDER, -0 < +0 is well-defined so no expansion needed.
+ if (!type.columnOrder().equals(ColumnOrder.ieee754TotalOrder())) {
+ if (min == (short) 0x0000) {
+ bMin = Float16.NEGATIVE_ZERO_LITTLE_ENDIAN;
+ stats.setMinMax(bMin, bMax);
+ }
+ if (max == (short) 0x8000) {
+ bMax = Float16.POSITIVE_ZERO_LITTLE_ENDIAN;
+ stats.setMinMax(bMin, bMax);
+ }
}
}
}
@@ -180,6 +203,7 @@ public abstract class Statistics<T extends Comparable<T>> {
private final PrimitiveComparator<T> comparator;
private boolean hasNonNullValue;
private long num_nulls;
+ private long nan_count = -1;
final PrimitiveStringifier stringifier;
Statistics(PrimitiveType type) {
@@ -236,14 +260,24 @@ public abstract class Statistics<T extends Comparable<T>> {
case INT64:
return new LongStatistics(primitive);
case FLOAT:
- return new FloatStatistics(primitive);
+ return primitive.columnOrder().equals(ColumnOrder.ieee754TotalOrder())
+ ? new IEEE754FloatStatistics(primitive)
+ : new FloatStatistics(primitive);
case DOUBLE:
- return new DoubleStatistics(primitive);
+ return primitive.columnOrder().equals(ColumnOrder.ieee754TotalOrder())
+ ? new IEEE754DoubleStatistics(primitive)
+ : new DoubleStatistics(primitive);
case BOOLEAN:
return new BooleanStatistics(primitive);
case BINARY:
case INT96:
case FIXED_LEN_BYTE_ARRAY:
+ if (primitive.getLogicalTypeAnnotation()
+ instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation) {
+ return primitive.columnOrder().equals(ColumnOrder.ieee754TotalOrder())
+ ? new IEEE754Float16Statistics(primitive)
+ : new Float16Statistics(primitive);
+ }
return new BinaryStatistics(primitive);
default:
throw new UnknownColumnTypeException(primitive.getPrimitiveTypeName());
@@ -349,7 +383,8 @@ public abstract class Statistics<T extends Comparable<T>> {
return type.equals(stats.type)
&& Arrays.equals(stats.getMaxBytes(), this.getMaxBytes())
&& Arrays.equals(stats.getMinBytes(), this.getMinBytes())
- && stats.getNumNulls() == this.getNumNulls();
+ && stats.getNumNulls() == this.getNumNulls()
+ && stats.getNanCount() == this.getNanCount();
}
/**
@@ -362,6 +397,7 @@ public abstract class Statistics<T extends Comparable<T>> {
return 31 * type.hashCode()
+ 31 * Arrays.hashCode(getMaxBytes())
+ 17 * Arrays.hashCode(getMinBytes())
+ + 13 * Long.valueOf(this.getNanCount()).hashCode()
+ Long.valueOf(this.getNumNulls()).hashCode();
}
@@ -382,6 +418,11 @@ public abstract class Statistics<T extends Comparable<T>> {
mergeStatisticsMinMax(stats);
markAsNotEmpty();
}
+ if (isNanCountSet() && stats.isNanCountSet()) {
+ incrementNanCount(stats.getNanCount());
+ } else {
+ unsetNanCount();
+ }
} else {
throw StatisticsClassException.create(this, stats);
}
@@ -533,6 +574,53 @@ public abstract class Statistics<T extends Comparable<T>> {
num_nulls += increment;
}
+ /**
+ * Increments the NaN count by one. If nan_count was not set (-1), initializes it to 1.
+ */
+ public void incrementNanCount() {
+ if (nan_count < 0) {
+ nan_count = 1;
+ } else {
+ nan_count++;
+ }
+ }
+
+ /**
+ * Increments the NaN count by the parameter value. If nan_count was not set (-1), initializes it to increment.
+ *
+ * @param increment value to increment the NaN count by
+ */
+ public void incrementNanCount(long increment) {
+ if (nan_count < 0) {
+ nan_count = increment;
+ } else {
+ nan_count += increment;
+ }
+ }
+
+ /**
+ * Returns the NaN count
+ *
+ * @return NaN count or {@code -1} if the NaN count is not set
+ */
+ public long getNanCount() {
+ return nan_count;
+ }
+
+ /**
+ * @return whether nanCount is set and can be used
+ */
+ public boolean isNanCountSet() {
+ return nan_count >= 0;
+ }
+
+ /**
+ * Unsets the NaN count to -1.
+ */
+ public void unsetNanCount() {
+ nan_count = -1;
+ }
+
/**
* Returns the null count
*
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesWriter.java
index c197a4fd6..077be9f65 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesWriter.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesWriter.java
@@ -116,7 +116,7 @@ public abstract class ByteStreamSplitValuesWriter extends ValuesWriter {
@Override
public void writeFloat(float v) {
- super.scatterBytes(BytesUtils.intToBytes(Float.floatToIntBits(v)));
+ super.scatterBytes(BytesUtils.intToBytes(Float.floatToRawIntBits(v)));
}
@Override
@@ -133,7 +133,7 @@ public abstract class ByteStreamSplitValuesWriter extends ValuesWriter {
@Override
public void writeDouble(double v) {
- super.scatterBytes(BytesUtils.longToBytes(Double.doubleToLongBits(v)));
+ super.scatterBytes(BytesUtils.longToBytes(Double.doubleToRawLongBits(v)));
}
@Override
diff --git a/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesWriter.java b/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesWriter.java
index 53526ae8d..92e88bac9 100644
--- a/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesWriter.java
+++ b/parquet-column/src/main/java/org/apache/parquet/column/values/dictionary/DictionaryValuesWriter.java
@@ -20,12 +20,6 @@ package org.apache.parquet.column.values.dictionary;
import static org.apache.parquet.bytes.BytesInput.concat;
-import it.unimi.dsi.fastutil.doubles.Double2IntLinkedOpenHashMap;
-import it.unimi.dsi.fastutil.doubles.Double2IntMap;
-import it.unimi.dsi.fastutil.doubles.DoubleIterator;
-import it.unimi.dsi.fastutil.floats.Float2IntLinkedOpenHashMap;
-import it.unimi.dsi.fastutil.floats.Float2IntMap;
-import it.unimi.dsi.fastutil.floats.FloatIterator;
import it.unimi.dsi.fastutil.ints.Int2IntLinkedOpenHashMap;
import it.unimi.dsi.fastutil.ints.Int2IntMap;
import it.unimi.dsi.fastutil.longs.Long2IntLinkedOpenHashMap;
@@ -417,7 +411,7 @@ public abstract class DictionaryValuesWriter extends ValuesWriter implements Req
public static class PlainDoubleDictionaryValuesWriter extends DictionaryValuesWriter {
/* type specific dictionary content */
- private Double2IntMap doubleDictionaryContent = new Double2IntLinkedOpenHashMap();
+ private Long2IntMap doubleDictionaryContent = new Long2IntLinkedOpenHashMap();
public PlainDoubleDictionaryValuesWriter(
int maxDictionaryByteSize,
@@ -430,10 +424,11 @@ public abstract class DictionaryValuesWriter extends ValuesWriter implements Req
@Override
public void writeDouble(double v) {
- int id = doubleDictionaryContent.get(v);
+ long bits = Double.doubleToRawLongBits(v);
+ int id = doubleDictionaryContent.get(bits);
if (id == -1) {
id = doubleDictionaryContent.size();
- doubleDictionaryContent.put(v, id);
+ doubleDictionaryContent.put(bits, id);
dictionaryByteSize += 8;
}
encodedValues.add(id);
@@ -445,10 +440,10 @@ public abstract class DictionaryValuesWriter extends ValuesWriter implements Req
// return a dictionary only if we actually used it
PlainValuesWriter dictionaryEncoder =
new PlainValuesWriter(lastUsedDictionaryByteSize, maxDictionaryByteSize, allocator);
- DoubleIterator doubleIterator = doubleDictionaryContent.keySet().iterator();
+ LongIterator doubleIterator = doubleDictionaryContent.keySet().iterator();
// write only the part of the dict that we used
for (int i = 0; i < lastUsedDictionarySize; i++) {
- dictionaryEncoder.writeDouble(doubleIterator.nextDouble());
+ dictionaryEncoder.writeDouble(Double.longBitsToDouble(doubleIterator.nextLong()));
}
return dictPage(dictionaryEncoder);
}
@@ -468,21 +463,24 @@ public abstract class DictionaryValuesWriter extends ValuesWriter implements Req
@Override
public void fallBackDictionaryEncodedData(ValuesWriter writer) {
// build reverse dictionary
- double[] reverseDictionary = new double[getDictionarySize()];
- ObjectIterator<Double2IntMap.Entry> entryIterator =
- doubleDictionaryContent.double2IntEntrySet().iterator();
+ long[] reverseDictionary = new long[getDictionarySize()];
+ ObjectIterator<Long2IntMap.Entry> entryIterator = longDictionaryContentIterator();
while (entryIterator.hasNext()) {
- Double2IntMap.Entry entry = entryIterator.next();
- reverseDictionary[entry.getIntValue()] = entry.getDoubleKey();
+ Long2IntMap.Entry entry = entryIterator.next();
+ reverseDictionary[entry.getIntValue()] = entry.getLongKey();
}
// fall back to plain encoding
IntIterator iterator = encodedValues.iterator();
while (iterator.hasNext()) {
int id = iterator.next();
- writer.writeDouble(reverseDictionary[id]);
+ writer.writeDouble(Double.longBitsToDouble(reverseDictionary[id]));
}
}
+
+ private ObjectIterator<Long2IntMap.Entry> longDictionaryContentIterator() {
+ return doubleDictionaryContent.long2IntEntrySet().iterator();
+ }
}
public static class PlainIntegerDictionaryValuesWriter extends DictionaryValuesWriter {
@@ -560,7 +558,7 @@ public abstract class DictionaryValuesWriter extends ValuesWriter implements Req
public static class PlainFloatDictionaryValuesWriter extends DictionaryValuesWriter {
/* type specific dictionary content */
- private Float2IntMap floatDictionaryContent = new Float2IntLinkedOpenHashMap();
+ private Int2IntMap floatDictionaryContent = new Int2IntLinkedOpenHashMap();
public PlainFloatDictionaryValuesWriter(
int maxDictionaryByteSize,
@@ -573,10 +571,11 @@ public abstract class DictionaryValuesWriter extends ValuesWriter implements Req
@Override
public void writeFloat(float v) {
- int id = floatDictionaryContent.get(v);
+ int bits = Float.floatToRawIntBits(v);
+ int id = floatDictionaryContent.get(bits);
if (id == -1) {
id = floatDictionaryContent.size();
- floatDictionaryContent.put(v, id);
+ floatDictionaryContent.put(bits, id);
dictionaryByteSize += 4;
}
encodedValues.add(id);
@@ -588,10 +587,11 @@ public abstract class DictionaryValuesWriter extends ValuesWriter implements Req
// return a dictionary only if we actually used it
PlainValuesWriter dictionaryEncoder =
new PlainValuesWriter(lastUsedDictionaryByteSize, maxDictionaryByteSize, allocator);
- FloatIterator floatIterator = floatDictionaryContent.keySet().iterator();
+ it.unimi.dsi.fastutil.ints.IntIterator floatIterator =
+ floatDictionaryContent.keySet().iterator();
// write only the part of the dict that we used
for (int i = 0; i < lastUsedDictionarySize; i++) {
- dictionaryEncoder.writeFloat(floatIterator.nextFloat());
+ dictionaryEncoder.writeFloat(Float.intBitsToFloat(floatIterator.nextInt()));
}
return dictPage(dictionaryEncoder);
}
@@ -611,19 +611,19 @@ public abstract class DictionaryValuesWriter extends ValuesWriter implements Req
@Override
public void fallBackDictionaryEncodedData(ValuesWriter writer) {
// build reverse dictionary
- float[] reverseDictionary = new float[getDictionarySize()];
- ObjectIterator<Float2IntMap.Entry> entryIterator =
- floatDictionaryContent.float2IntEntrySet().iterator();
+ int[] reverseDictionary = new int[getDictionarySize()];
+ ObjectIterator<Int2IntMap.Entry> entryIterator =
+ floatDictionaryContent.int2IntEntrySet().iterator();
while (entryIterator.hasNext()) {
- Float2IntMap.Entry entry = entryIterator.next();
- reverseDictionary[entry.getIntValue()] = entry.getFloatKey();
+ Int2IntMap.Entry entry = entryIterator.next();
+ reverseDictionary[entry.getIntValue()] = entry.getIntKey();
}
// fall back to plain encoding
IntIterator iterator = encodedValues.iterator();
while (iterator.hasNext()) {
int id = iterator.next();
- writer.writeFloat(reverseDictionary[id]);
+ writer.writeFloat(Float.intBitsToFloat(reverseDictionary[id]));
}
}
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java
index 24de97d01..9dcbbadf7 100644
--- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/BinaryColumnIndexBuilder.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.parquet.filter2.predicate.Statistics;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
import org.apache.parquet.schema.Float16;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.PrimitiveComparator;
@@ -32,9 +33,41 @@ class BinaryColumnIndexBuilder extends ColumnIndexBuilder {
private static class BinaryColumnIndex extends ColumnIndexBase<Binary> {
private Binary[] minValues;
private Binary[] maxValues;
+ private final boolean isFloat16;
+ private final boolean isIeee754TotalOrder;
private BinaryColumnIndex(PrimitiveType type) {
super(type);
+ isFloat16 = type.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
+ isIeee754TotalOrder = type.columnOrder().equals(ColumnOrder.ieee754TotalOrder());
+ }
+
+ @Override
+ boolean mayHaveNaNPollutedMinMax() {
+ return isFloat16 && !isIeee754TotalOrder;
+ }
+
+ @Override
+ boolean isNaNLiteral(Object value) {
+ return isFloat16
+ && value instanceof Binary
+ && ((Binary) value).length() == LogicalTypeAnnotation.Float16LogicalTypeAnnotation.BYTES
+ && Float16.isNaN(((Binary) value).get2BytesLittleEndian());
+ }
+
+ @Override
+ boolean isMinNaN(int arrayIndex) {
+ return isFloat16 && Float16.isNaN(minValues[arrayIndex].get2BytesLittleEndian());
+ }
+
+ @Override
+ boolean isMaxNaN(int arrayIndex) {
+ return isFloat16 && Float16.isNaN(maxValues[arrayIndex].get2BytesLittleEndian());
+ }
+
+ @Override
+ boolean hasNaNs(int pageIndex) {
+ return isFloat16 && (nanCounts == null || nanCounts[pageIndex] > 0);
}
@Override
@@ -85,6 +118,7 @@ class BinaryColumnIndexBuilder extends ColumnIndexBuilder {
private final BinaryTruncator truncator;
private final int truncateLength;
private final boolean isFloat16;
+ private final boolean isIeee754TotalOrder;
private boolean invalid;
private static Binary convert(ByteBuffer buffer) {
@@ -99,6 +133,7 @@ class BinaryColumnIndexBuilder extends ColumnIndexBuilder {
truncator = BinaryTruncator.getTruncator(type);
this.truncateLength = truncateLength;
this.isFloat16 = type.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
+ this.isIeee754TotalOrder = type.columnOrder().equals(ColumnOrder.ieee754TotalOrder());
}
@Override
@@ -122,17 +157,21 @@ class BinaryColumnIndexBuilder extends ColumnIndexBuilder {
short sMax = bMax.get2BytesLittleEndian();
if (Float16.isNaN(sMin) || Float16.isNaN(sMax)) {
- invalid = true;
+ if (!isIeee754TotalOrder) {
+ invalid = true;
+ }
}
- // Sorting order is undefined for -0.0 so let min = -0.0 and max = +0.0 to
- // ensure that no 0.0 values are skipped
+ // For TYPE_DEFINED_ORDER, sorting order is undefined for -0.0 so let min = -0.0 and max = +0.0
+ // to ensure that no 0.0 values are skipped. For IEEE_754_TOTAL_ORDER, -0 < +0 is well-defined.
// +0.0 is 0x0000, -0.0 is 0x8000 (little endian: 00 00, 00 80)
- if (sMin == (short) 0x0000) {
- bMin = Float16.NEGATIVE_ZERO_LITTLE_ENDIAN;
- }
- if (sMax == (short) 0x8000) {
- bMax = Float16.POSITIVE_ZERO_LITTLE_ENDIAN;
+ if (!isIeee754TotalOrder) {
+ if (sMin == (short) 0x0000) {
+ bMin = Float16.NEGATIVE_ZERO_LITTLE_ENDIAN;
+ }
+ if (sMax == (short) 0x8000) {
+ bMax = Float16.POSITIVE_ZERO_LITTLE_ENDIAN;
+ }
}
}
}
@@ -141,6 +180,13 @@ class BinaryColumnIndexBuilder extends ColumnIndexBuilder {
maxValues.add(bMax == null ? null : truncator.truncateMax(bMax, truncateLength));
}
+ @Override
+ void onNanEncountered() {
+ if (isFloat16 && !isIeee754TotalOrder) {
+ invalid = true;
+ }
+ }
+
@Override
ColumnIndexBase<Binary> createColumnIndex(PrimitiveType type) {
if (invalid) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndex.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndex.java
index 86099717d..1cf73784c 100644
--- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndex.java
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndex.java
@@ -71,4 +71,12 @@ public interface ColumnIndex extends Visitor<PrimitiveIterator.OfInt> {
default List<Long> getDefinitionLevelHistogram() {
throw new UnsupportedOperationException("Definition level histogram is not implemented");
}
+
+ /**
+ * @return the unmodifiable list of NaN counts for each page, or {@code null} if NaN counts are not available;
+ * used for converting to the related thrift object
+ */
+ default List<Long> getNanCounts() {
+ return null;
+ }
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java
index bd9073e26..d252f6068 100644
--- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/ColumnIndexBuilder.java
@@ -32,6 +32,7 @@ import it.unimi.dsi.fastutil.longs.LongList;
import it.unimi.dsi.fastutil.longs.LongLists;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Formatter;
import java.util.List;
import java.util.PrimitiveIterator;
@@ -102,6 +103,8 @@ public abstract class ColumnIndexBuilder {
// might be null
private long[] nullCounts;
// might be null
+ long[] nanCounts;
+ // might be null
private long[] repLevelHistogram;
// might be null
private long[] defLevelHistogram;
@@ -133,6 +136,14 @@ public abstract class ColumnIndexBuilder {
return LongLists.unmodifiable(LongArrayList.wrap(nullCounts));
}
+ @Override
+ public List<Long> getNanCounts() {
+ if (nanCounts == null) {
+ return null;
+ }
+ return LongLists.unmodifiable(LongArrayList.wrap(nanCounts));
+ }
+
@Override
public List<Boolean> getNullPages() {
return BooleanLists.unmodifiable(BooleanArrayList.wrap(nullPages));
@@ -242,6 +253,41 @@ public abstract class ColumnIndexBuilder {
return nullPages[pageIndex];
}
+ // Returns true if this ColumnIndex may have NaN-polluted min/max values that cannot be trusted.
+ // If true, we need to conservatively include pages with NaN in min/max for some predicates.
+ boolean mayHaveNaNPollutedMinMax() {
+ return false;
+ }
+
+ // Returns true if the given predicate value is NaN for this column's type. Override in typed subclasses.
+ boolean isNaNLiteral(Object value) {
+ return false;
+ }
+
+ // Returns true if the min value at arrayIndex is NaN. Override in typed subclasses.
+ boolean isMinNaN(int arrayIndex) {
+ return false;
+ }
+
+ // Returns true if the max value at arrayIndex is NaN. Override in typed subclasses.
+ boolean isMaxNaN(int arrayIndex) {
+ return false;
+ }
+
+ // Returns true if the min or max value at arrayIndex is NaN.
+ boolean hasNaNMinMax(int arrayIndex) {
+ return isMinNaN(arrayIndex) || isMaxNaN(arrayIndex);
+ }
+
+ boolean pageHasNaNMinMax(int pageIndex) {
+ int arrayIndex = Arrays.binarySearch(pageIndexes, pageIndex);
+ return arrayIndex >= 0 && hasNaNMinMax(arrayIndex);
+ }
+
+ boolean hasNaNs(int pageIndex) {
+ return false;
+ }
+
/*
* Returns the min value for arrayIndex as a ByteBuffer. (Min values are not stored for null-pages so arrayIndex
* might not equal to pageIndex.)
@@ -273,6 +319,18 @@ public abstract class ColumnIndexBuilder {
/* Creates a ValueComparator object containing the specified value to be compared for min/max values */
abstract ValueComparator createValueComparator(Object value);
+ private PrimitiveIterator.OfInt includeMatchingPages(
+ PrimitiveIterator.OfInt result, IntPredicate extraPageMatcher) {
+ IntSet pages = new IntOpenHashSet();
+ result.forEachRemaining((int pageIndex) -> pages.add(pageIndex));
+ for (int pageIndex = 0; pageIndex < getPageCount(); pageIndex++) {
+ if (extraPageMatcher.test(pageIndex)) {
+ pages.add(pageIndex);
+ }
+ }
+ return IndexIterator.filter(getPageCount(), pages::contains);
+ }
+
@Override
public PrimitiveIterator.OfInt visit(And and) {
throw new UnsupportedOperationException("AND shall not be used on column index directly");
@@ -299,27 +357,72 @@ public abstract class ColumnIndexBuilder {
return IndexIterator.filter(getPageCount(), pageIndex -> nullCounts[pageIndex] > 0);
}
}
+ if (isNaNLiteral(value)) {
+ return nanCounts == null
+ ? IndexIterator.all(getPageCount())
+ : IndexIterator.filter(getPageCount(), pageIndex -> nanCounts[pageIndex] > 0);
+ }
+ if (mayHaveNaNPollutedMinMax()) {
+ return includeMatchingPages(
+ getBoundaryOrder().eq(createValueComparator(value)), this::pageHasNaNMinMax);
+ }
return getBoundaryOrder().eq(createValueComparator(value));
}
@Override
public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(Gt<T> gt) {
- return getBoundaryOrder().gt(createValueComparator(gt.getValue()));
+ T value = gt.getValue();
+ if (isNaNLiteral(value)) {
+ return IndexIterator.all(getPageCount());
+ }
+ if (mayHaveNaNPollutedMinMax()) {
+ return includeMatchingPages(
+ getBoundaryOrder().gt(createValueComparator(value)),
+ pageIndex -> hasNaNs(pageIndex) || pageHasNaNMinMax(pageIndex));
+ }
+ return includeMatchingPages(getBoundaryOrder().gt(createValueComparator(value)), this::hasNaNs);
}
@Override
public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(GtEq<T> gtEq) {
- return getBoundaryOrder().gtEq(createValueComparator(gtEq.getValue()));
+ T value = gtEq.getValue();
+ if (isNaNLiteral(value)) {
+ return IndexIterator.all(getPageCount());
+ }
+ if (mayHaveNaNPollutedMinMax()) {
+ return includeMatchingPages(
+ getBoundaryOrder().gtEq(createValueComparator(value)),
+ pageIndex -> hasNaNs(pageIndex) || pageHasNaNMinMax(pageIndex));
+ }
+ return includeMatchingPages(getBoundaryOrder().gtEq(createValueComparator(value)), this::hasNaNs);
}
@Override
public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(Lt<T> lt) {
- return getBoundaryOrder().lt(createValueComparator(lt.getValue()));
+ T value = lt.getValue();
+ if (isNaNLiteral(value)) {
+ return IndexIterator.all(getPageCount());
+ }
+ if (mayHaveNaNPollutedMinMax()) {
+ return includeMatchingPages(
+ getBoundaryOrder().lt(createValueComparator(value)),
+ pageIndex -> hasNaNs(pageIndex) || pageHasNaNMinMax(pageIndex));
+ }
+ return includeMatchingPages(getBoundaryOrder().lt(createValueComparator(value)), this::hasNaNs);
}
@Override
public <T extends Comparable<T>> PrimitiveIterator.OfInt visit(LtEq<T> ltEq) {
- return getBoundaryOrder().ltEq(createValueComparator(ltEq.getValue()));
+ T value = ltEq.getValue();
+ if (isNaNLiteral(value)) {
+ return IndexIterator.all(getPageCount());
+ }
+ if (mayHaveNaNPollutedMinMax()) {
+ return includeMatchingPages(
+ getBoundaryOrder().ltEq(createValueComparator(value)),
+ pageIndex -> hasNaNs(pageIndex) || pageHasNaNMinMax(pageIndex));
+ }
+ return includeMatchingPages(getBoundaryOrder().ltEq(createValueComparator(value)), this::hasNaNs);
}
@Override
@@ -329,6 +432,15 @@ public abstract class ColumnIndexBuilder {
return IndexIterator.filter(getPageCount(), pageIndex -> !nullPages[pageIndex]);
}
+ if (isNaNLiteral(value)) {
+ // v != NaN is satisfied by every null value, every non-NaN value, and (under IEEE754 total
+ // order) every NaN whose bit pattern differs from the literal. The page-level min/max and
+ // nan_count are not sufficient to prove that *all* values of a page equal the NaN literal
+ // (e.g. a page may mix NaNs with nulls, or hold NaNs with different payloads), so we
+ // conservatively keep every page.
+ return IndexIterator.all(getPageCount());
+ }
+
if (nullCounts == null) {
// Nulls match so if we don't have null related statistics we have to return all pages
return IndexIterator.all(getPageCount());
@@ -336,11 +448,18 @@ public abstract class ColumnIndexBuilder {
// Merging value filtering with pages containing nulls
IntSet matchingIndexes = new IntOpenHashSet();
- getBoundaryOrder()
- .notEq(createValueComparator(value))
- .forEachRemaining((int index) -> matchingIndexes.add(index));
+ if (mayHaveNaNPollutedMinMax()) {
+ includeMatchingPages(getBoundaryOrder().notEq(createValueComparator(value)), this::pageHasNaNMinMax)
+ .forEachRemaining((int index) -> matchingIndexes.add(index));
+ } else {
+ getBoundaryOrder()
+ .notEq(createValueComparator(value))
+ .forEachRemaining((int index) -> matchingIndexes.add(index));
+ }
return IndexIterator.filter(
- getPageCount(), pageIndex -> nullCounts[pageIndex] > 0 || matchingIndexes.contains(pageIndex));
+ getPageCount(),
+ pageIndex ->
+ nullCounts[pageIndex] > 0 || hasNaNs(pageIndex) || matchingIndexes.contains(pageIndex));
}
@Override
@@ -362,6 +481,19 @@ public abstract class ColumnIndexBuilder {
return IndexIterator.filter(getPageCount(), matchingIndexesForNull::contains);
}
}
+ } else if (isNaNLiteral(value)) {
+ if (nanCounts == null || values.size() != 1) {
+ // We don't know if NaN exists in any page, or we can't rely on ltEq/gtEq on NaN, return all
+ // pages.
+ return IndexIterator.all(getPageCount());
+ }
+ IntSet matchingIndexesForNaN = new IntOpenHashSet();
+ for (int i = 0; i < nanCounts.length; i++) {
+ if (nanCounts[i] > 0) {
+ matchingIndexesForNaN.add(i);
+ }
+ }
+ return IndexIterator.filter(getPageCount(), matchingIndexesForNaN::contains);
}
}
@@ -425,6 +557,9 @@ public abstract class ColumnIndexBuilder {
return acceptNulls;
} else {
++arrayIndex;
+ if (hasNaNs(pageIndex)) {
+ return true;
+ }
if (acceptNulls && nullCounts[pageIndex] > 0) {
return true;
}
@@ -517,10 +652,12 @@ public abstract class ColumnIndexBuilder {
private PrimitiveType type;
private final BooleanList nullPages = new BooleanArrayList();
private final LongList nullCounts = new LongArrayList();
+ private LongList nanCounts = new LongArrayList();
private final IntList pageIndexes = new IntArrayList();
private int nextPageIndex;
private LongList repLevelHistogram = new LongArrayList();
private LongList defLevelHistogram = new LongArrayList();
+ private boolean hasNonNullPagesWithMissingMinMax;
/**
* @return a no-op builder that does not collect statistics objects and therefore returns {@code null} at
@@ -550,9 +687,9 @@ public abstract class ColumnIndexBuilder {
case BOOLEAN:
return new BooleanColumnIndexBuilder();
case DOUBLE:
- return new DoubleColumnIndexBuilder();
+ return new DoubleColumnIndexBuilder(type);
case FLOAT:
- return new FloatColumnIndexBuilder();
+ return new FloatColumnIndexBuilder(type);
case INT32:
return new IntColumnIndexBuilder();
case INT64:
@@ -611,10 +748,53 @@ public abstract class ColumnIndexBuilder {
List<ByteBuffer> maxValues,
List<Long> repLevelHistogram,
List<Long> defLevelHistogram) {
+ return build(
+ type,
+ boundaryOrder,
+ nullPages,
+ nullCounts,
+ null,
+ minValues,
+ maxValues,
+ repLevelHistogram,
+ defLevelHistogram);
+ }
- ColumnIndexBuilder builder = createNewBuilder(type, Integer.MAX_VALUE);
+ /**
+ * @param type
+ * the primitive type
+ * @param boundaryOrder
+ * the boundary order of the min/max values
+ * @param nullPages
+ * the null pages (one boolean value for each page that signifies whether the page consists of nulls
+ * entirely)
+ * @param nullCounts
+ * the number of null values for each page
+ * @param nanCounts
+ * the number of NaN values for each page (may be null)
+ * @param minValues
+ * the min values for each page
+ * @param maxValues
+ * the max values for each page
+ * @param repLevelHistogram
+ * the repetition level histogram for all levels of each page
+ * @param defLevelHistogram
+ * the definition level histogram for all levels of each page
+ * @return the newly created {@link ColumnIndex} object based on the specified arguments
+ */
+ public static ColumnIndex build(
+ PrimitiveType type,
+ BoundaryOrder boundaryOrder,
+ List<Boolean> nullPages,
+ List<Long> nullCounts,
+ List<Long> nanCounts,
+ List<ByteBuffer> minValues,
+ List<ByteBuffer> maxValues,
+ List<Long> repLevelHistogram,
+ List<Long> defLevelHistogram) {
- builder.fill(nullPages, nullCounts, minValues, maxValues, repLevelHistogram, defLevelHistogram);
+ ColumnIndexBuilder builder = createNewBuilder(type, Integer.MAX_VALUE);
+ builder.fill(nullPages, nullCounts, nanCounts, minValues, maxValues, repLevelHistogram, defLevelHistogram);
ColumnIndexBase<?> columnIndex = builder.build(type);
if (columnIndex == null) {
return null;
@@ -653,9 +833,21 @@ public abstract class ColumnIndexBuilder {
pageIndexes.add(nextPageIndex);
} else {
nullPages.add(true);
+ if (stats.isNanCountSet() && stats.getNanCount() > 0) {
+ hasNonNullPagesWithMissingMinMax = true;
+ }
}
nullCounts.add(stats.getNumNulls());
+ if (nanCounts != null && stats.isNanCountSet()) {
+ nanCounts.add(stats.getNanCount());
+ if (stats.getNanCount() > 0) {
+ onNanEncountered();
+ }
+ } else {
+ nanCounts = null;
+ }
+
// Collect repetition and definition level histograms only when all pages are valid.
if (sizeStats != null && sizeStats.isValid() && repLevelHistogram != null && defLevelHistogram != null) {
repLevelHistogram.addAll(sizeStats.getRepetitionLevelHistogram());
@@ -672,9 +864,18 @@ public abstract class ColumnIndexBuilder {
abstract void addMinMax(Object min, Object max);
+ /**
+ * Called when a page with NaN values is encountered (nan_count > 0).
+ * Subclasses should override to handle NaN presence (e.g., invalidate for TYPE_DEFINED_ORDER).
+ */
+ void onNanEncountered() {
+ throw new UnsupportedOperationException("Cannot call onNanEncountered on type: " + type);
+ }
+
private void fill(
List<Boolean> nullPages,
List<Long> nullCounts,
+ List<Long> nanCounts,
List<ByteBuffer> minValues,
List<ByteBuffer> maxValues,
List<Long> repLevelHistogram,
@@ -682,12 +883,14 @@ public abstract class ColumnIndexBuilder {
clear();
int pageCount = nullPages.size();
if ((nullCounts != null && nullCounts.size() != pageCount)
+ || (nanCounts != null && nanCounts.size() != pageCount)
|| minValues.size() != pageCount
|| maxValues.size() != pageCount) {
throw new IllegalArgumentException(String.format(
- "Not all sizes are equal (nullPages:%d, nullCounts:%s, minValues:%d, maxValues:%d",
+ "Not all sizes are equal (nullPages:%d, nullCounts:%s, nanCounts:%s, minValues:%d, maxValues:%d",
nullPages.size(),
nullCounts == null ? "null" : nullCounts.size(),
+ nanCounts == null ? "null" : nanCounts.size(),
minValues.size(),
maxValues.size()));
}
@@ -708,6 +911,12 @@ public abstract class ColumnIndexBuilder {
if (nullCounts != null) {
this.nullCounts.addAll(nullCounts);
}
+ // NaN counts is optional in the format
+ if (nanCounts != null) {
+ this.nanCounts.addAll(nanCounts);
+ } else {
+ this.nanCounts = null;
+ }
for (int i = 0; i < pageCount; ++i) {
if (!nullPages.get(i)) {
@@ -743,6 +952,9 @@ public abstract class ColumnIndexBuilder {
if (nullPages.isEmpty()) {
return null;
}
+ if (hasNonNullPagesWithMissingMinMax) {
+ return null;
+ }
ColumnIndexBase<?> columnIndex = createColumnIndex(type);
if (columnIndex == null) {
// Might happen if the specialized builder discovers invalid min/max values
@@ -766,6 +978,10 @@ public abstract class ColumnIndexBuilder {
}
}
}
+ // NaN counts is optional so keep it null if the builder has no values
+ if (nanCounts != null && !nanCounts.isEmpty()) {
+ columnIndex.nanCounts = nanCounts.toLongArray();
+ }
columnIndex.pageIndexes = pageIndexes.toIntArray();
// Repetition and definition level histograms are optional so keep them null if the builder has no values
if (repLevelHistogram != null && !repLevelHistogram.isEmpty()) {
@@ -817,9 +1033,23 @@ public abstract class ColumnIndexBuilder {
nullCounts.clear();
clearMinMax();
nextPageIndex = 0;
+ hasNonNullPagesWithMissingMinMax = false;
pageIndexes.clear();
- repLevelHistogram.clear();
- defLevelHistogram.clear();
+ if (nanCounts != null) {
+ nanCounts.clear();
+ } else {
+ nanCounts = new LongArrayList();
+ }
+ if (repLevelHistogram == null) {
+ repLevelHistogram = new LongArrayList();
+ } else {
+ repLevelHistogram.clear();
+ }
+ if (defLevelHistogram == null) {
+ defLevelHistogram = new LongArrayList();
+ } else {
+ defLevelHistogram.clear();
+ }
}
abstract void clearMinMax();
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/DoubleColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/DoubleColumnIndexBuilder.java
index 5d5d54aa7..9f679e65e 100644
--- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/DoubleColumnIndexBuilder.java
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/DoubleColumnIndexBuilder.java
@@ -25,6 +25,7 @@ import it.unimi.dsi.fastutil.doubles.DoubleList;
import java.nio.ByteBuffer;
import org.apache.parquet.filter2.predicate.Statistics;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
import org.apache.parquet.schema.PrimitiveComparator;
import org.apache.parquet.schema.PrimitiveType;
@@ -32,9 +33,36 @@ class DoubleColumnIndexBuilder extends ColumnIndexBuilder {
private static class DoubleColumnIndex extends ColumnIndexBase<Double> {
private double[] minValues;
private double[] maxValues;
+ private final boolean isIeee754TotalOrder;
private DoubleColumnIndex(PrimitiveType type) {
super(type);
+ isIeee754TotalOrder = type.columnOrder().equals(ColumnOrder.ieee754TotalOrder());
+ }
+
+ @Override
+ boolean mayHaveNaNPollutedMinMax() {
+ return !isIeee754TotalOrder;
+ }
+
+ @Override
+ boolean isNaNLiteral(Object value) {
+ return Double.isNaN((double) value);
+ }
+
+ @Override
+ boolean isMinNaN(int arrayIndex) {
+ return Double.isNaN(minValues[arrayIndex]);
+ }
+
+ @Override
+ boolean isMaxNaN(int arrayIndex) {
+ return Double.isNaN(maxValues[arrayIndex]);
+ }
+
+ @Override
+ boolean hasNaNs(int pageIndex) {
+ return nanCounts == null || nanCounts[pageIndex] > 0;
}
@Override
@@ -83,6 +111,11 @@ class DoubleColumnIndexBuilder extends ColumnIndexBuilder {
private final DoubleList minValues = new DoubleArrayList();
private final DoubleList maxValues = new DoubleArrayList();
private boolean invalid;
+ private final boolean isIeee754TotalOrder;
+
+ DoubleColumnIndexBuilder(PrimitiveType type) {
+ this.isIeee754TotalOrder = type.columnOrder().equals(ColumnOrder.ieee754TotalOrder());
+ }
private static double convert(ByteBuffer buffer) {
return buffer.order(LITTLE_ENDIAN).getDouble(0);
@@ -103,22 +136,33 @@ class DoubleColumnIndexBuilder extends ColumnIndexBuilder {
double dMin = (double) min;
double dMax = (double) max;
if (Double.isNaN(dMin) || Double.isNaN(dMax)) {
- // Invalidate this column index in case of NaN as the sorting order of values is undefined for this case
- invalid = true;
+ if (!isIeee754TotalOrder) {
+ invalid = true;
+ }
}
- // Sorting order is undefined for -0.0 so let min = -0.0 and max = +0.0 to ensure that no 0.0 values are skipped
- if (Double.compare(dMin, +0.0) == 0) {
- dMin = -0.0;
- }
- if (Double.compare(dMax, -0.0) == 0) {
- dMax = +0.0;
+ // For TYPE_DEFINED_ORDER, sorting order is undefined for -0.0 so let min = -0.0 and max = +0.0
+ // to ensure that no 0.0 values are skipped. For IEEE_754_TOTAL_ORDER, -0 < +0 is well-defined.
+ if (!isIeee754TotalOrder) {
+ if (Double.compare(dMin, +0.0) == 0) {
+ dMin = -0.0;
+ }
+ if (Double.compare(dMax, -0.0) == 0) {
+ dMax = +0.0;
+ }
}
minValues.add(dMin);
maxValues.add(dMax);
}
+ @Override
+ void onNanEncountered() {
+ if (!isIeee754TotalOrder) {
+ invalid = true;
+ }
+ }
+
@Override
ColumnIndexBase<Double> createColumnIndex(PrimitiveType type) {
if (invalid) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java
index be66f85d1..bf78c707e 100644
--- a/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java
+++ b/parquet-column/src/main/java/org/apache/parquet/internal/column/columnindex/FloatColumnIndexBuilder.java
@@ -25,6 +25,7 @@ import it.unimi.dsi.fastutil.floats.FloatList;
import java.nio.ByteBuffer;
import org.apache.parquet.filter2.predicate.Statistics;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
import org.apache.parquet.schema.PrimitiveComparator;
import org.apache.parquet.schema.PrimitiveType;
@@ -32,9 +33,36 @@ class FloatColumnIndexBuilder extends ColumnIndexBuilder {
private static class FloatColumnIndex extends ColumnIndexBase<Float> {
private float[] minValues;
private float[] maxValues;
+ private final boolean isIeee754TotalOrder;
private FloatColumnIndex(PrimitiveType type) {
super(type);
+ isIeee754TotalOrder = type.columnOrder().equals(ColumnOrder.ieee754TotalOrder());
+ }
+
+ @Override
+ boolean mayHaveNaNPollutedMinMax() {
+ return !isIeee754TotalOrder;
+ }
+
+ @Override
+ boolean isNaNLiteral(Object value) {
+ return Float.isNaN((float) value);
+ }
+
+ @Override
+ boolean isMinNaN(int arrayIndex) {
+ return Float.isNaN(minValues[arrayIndex]);
+ }
+
+ @Override
+ boolean isMaxNaN(int arrayIndex) {
+ return Float.isNaN(maxValues[arrayIndex]);
+ }
+
+ @Override
+ boolean hasNaNs(int pageIndex) {
+ return nanCounts == null || nanCounts[pageIndex] > 0;
}
@Override
@@ -83,6 +111,11 @@ class FloatColumnIndexBuilder extends ColumnIndexBuilder {
private final FloatList minValues = new FloatArrayList();
private final FloatList maxValues = new FloatArrayList();
private boolean invalid;
+ private final boolean isIeee754TotalOrder;
+
+ FloatColumnIndexBuilder(PrimitiveType type) {
+ this.isIeee754TotalOrder = type.columnOrder().equals(ColumnOrder.ieee754TotalOrder());
+ }
private static float convert(ByteBuffer buffer) {
return buffer.order(LITTLE_ENDIAN).getFloat(0);
@@ -103,22 +136,33 @@ class FloatColumnIndexBuilder extends ColumnIndexBuilder {
float fMin = (float) min;
float fMax = (float) max;
if (Float.isNaN(fMin) || Float.isNaN(fMax)) {
- // Invalidate this column index in case of NaN as the sorting order of values is undefined for this case
- invalid = true;
+ if (!isIeee754TotalOrder) {
+ invalid = true;
+ }
}
- // Sorting order is undefined for -0.0 so let min = -0.0 and max = +0.0 to ensure that no 0.0 values are skipped
- if (Float.compare(fMin, +0.0f) == 0) {
- fMin = -0.0f;
- }
- if (Float.compare(fMax, -0.0f) == 0) {
- fMax = +0.0f;
+ // For TYPE_DEFINED_ORDER, sorting order is undefined for -0.0 so let min = -0.0 and max = +0.0
+ // to ensure that no 0.0 values are skipped. For IEEE_754_TOTAL_ORDER, -0 < +0 is well-defined.
+ if (!isIeee754TotalOrder) {
+ if (Float.compare(fMin, +0.0f) == 0) {
+ fMin = -0.0f;
+ }
+ if (Float.compare(fMax, -0.0f) == 0) {
+ fMax = +0.0f;
+ }
}
minValues.add(fMin);
maxValues.add(fMax);
}
+ @Override
+ void onNanEncountered() {
+ if (!isIeee754TotalOrder) {
+ invalid = true;
+ }
+ }
+
@Override
ColumnIndexBase<Float> createColumnIndex(PrimitiveType type) {
if (invalid) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/ColumnOrder.java b/parquet-column/src/main/java/org/apache/parquet/schema/ColumnOrder.java
index 94a127556..35ef0ec9d 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/ColumnOrder.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/ColumnOrder.java
@@ -36,11 +36,16 @@ public class ColumnOrder {
/**
* Type defined order meaning that the comparison order of the elements are based on its type.
*/
- TYPE_DEFINED_ORDER
+ TYPE_DEFINED_ORDER,
+ /**
+ * The column order is defined by the IEEE 754 standard.
+ */
+ IEEE_754_TOTAL_ORDER,
}
private static final ColumnOrder UNDEFINED_COLUMN_ORDER = new ColumnOrder(ColumnOrderName.UNDEFINED);
private static final ColumnOrder TYPE_DEFINED_COLUMN_ORDER = new ColumnOrder(ColumnOrderName.TYPE_DEFINED_ORDER);
+ private static final ColumnOrder IEEE_754_TOTAL_ORDER = new ColumnOrder(ColumnOrderName.IEEE_754_TOTAL_ORDER);
/**
* @return a {@link ColumnOrder} instance representing an undefined order
@@ -58,6 +63,14 @@ public class ColumnOrder {
return TYPE_DEFINED_COLUMN_ORDER;
}
+ /**
+ * @return a {@link ColumnOrder} instance representing an IEEE 754 total order
+ * @see ColumnOrderName#IEEE_754_TOTAL_ORDER
+ */
+ public static ColumnOrder ieee754TotalOrder() {
+ return IEEE_754_TOTAL_ORDER;
+ }
+
private final ColumnOrderName columnOrderName;
private ColumnOrder(ColumnOrderName columnOrderName) {
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java b/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java
index 98bc5c023..625e9fd9d 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java
@@ -19,6 +19,7 @@
package org.apache.parquet.schema;
import static java.util.Optional.empty;
+import static org.apache.parquet.schema.ColumnOrder.ColumnOrderName.IEEE_754_TOTAL_ORDER;
import static org.apache.parquet.schema.ColumnOrder.ColumnOrderName.TYPE_DEFINED_ORDER;
import static org.apache.parquet.schema.ColumnOrder.ColumnOrderName.UNDEFINED;
import static org.apache.parquet.schema.PrimitiveStringifier.TIMESTAMP_MICROS_STRINGIFIER;
@@ -1047,6 +1048,13 @@ public abstract class LogicalTypeAnnotation {
PrimitiveStringifier valueStringifier(PrimitiveType primitiveType) {
return PrimitiveStringifier.FLOAT16_STRINGIFIER;
}
+
+ @Override
+ boolean isValidColumnOrder(ColumnOrder columnOrder) {
+ return columnOrder.getColumnOrderName() == UNDEFINED
+ || columnOrder.getColumnOrderName() == TYPE_DEFINED_ORDER
+ || columnOrder.getColumnOrderName() == IEEE_754_TOTAL_ORDER;
+ }
}
public static class UnknownLogicalTypeAnnotation extends LogicalTypeAnnotation {
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java
index 50c4acd4c..9d22d2531 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveComparator.java
@@ -293,4 +293,65 @@ public abstract class PrimitiveComparator<T> implements Comparator<T>, Serializa
return "BINARY_AS_FLOAT16_COMPARATOR";
}
};
+
+ static final PrimitiveComparator<Float> FLOAT_IEEE_754_TOTAL_ORDER_COMPARATOR = new PrimitiveComparator<Float>() {
+ @Override
+ int compareNotNulls(Float o1, Float o2) {
+ return compare(o1.floatValue(), o2.floatValue());
+ }
+
+ @Override
+ public int compare(float f1, float f2) {
+ int f1Int = Float.floatToRawIntBits(f1);
+ int f2Int = Float.floatToRawIntBits(f2);
+ f1Int ^= ((f1Int >> 31) >>> 1);
+ f2Int ^= ((f2Int >> 31) >>> 1);
+ return Integer.compare(f1Int, f2Int);
+ }
+
+ @Override
+ public String toString() {
+ return "FLOAT_IEEE_754_TOTAL_ORDER_COMPARATOR";
+ }
+ };
+
+ static final PrimitiveComparator<Double> DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR =
+ new PrimitiveComparator<Double>() {
+ @Override
+ int compareNotNulls(Double o1, Double o2) {
+ return compare(o1.doubleValue(), o2.doubleValue());
+ }
+
+ @Override
+ public int compare(double d1, double d2) {
+ long d1Long = Double.doubleToRawLongBits(d1);
+ long d2Long = Double.doubleToRawLongBits(d2);
+ d1Long ^= ((d1Long >> 63) >>> 1);
+ d2Long ^= ((d2Long >> 63) >>> 1);
+ return Long.compare(d1Long, d2Long);
+ }
+
+ @Override
+ public String toString() {
+ return "DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR";
+ }
+ };
+
+ static final PrimitiveComparator<Binary> BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR =
+ new BinaryComparator() {
+
+ @Override
+ int compareBinary(Binary b1, Binary b2) {
+ int b1Short = b1.get2BytesLittleEndian();
+ int b2Short = b2.get2BytesLittleEndian();
+ b1Short ^= ((b1Short >> 15) >>> 1);
+ b2Short ^= ((b2Short >> 15) >>> 1);
+ return Integer.compare(b1Short, b2Short);
+ }
+
+ @Override
+ public String toString() {
+ return "BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR";
+ }
+ };
}
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java
index 944cfb58e..81f2781cd 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java
@@ -88,7 +88,7 @@ public final class PrimitiveType extends Type {
}
@Override
- PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
+ PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
if (logicalType == null) {
return PrimitiveComparator.SIGNED_INT64_COMPARATOR;
}
@@ -152,7 +152,7 @@ public final class PrimitiveType extends Type {
}
@Override
- PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
+ PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
if (logicalType == null) {
return PrimitiveComparator.SIGNED_INT32_COMPARATOR;
}
@@ -222,7 +222,7 @@ public final class PrimitiveType extends Type {
}
@Override
- PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
+ PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
return PrimitiveComparator.BOOLEAN_COMPARATOR;
}
},
@@ -248,7 +248,7 @@ public final class PrimitiveType extends Type {
}
@Override
- PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
+ PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
if (logicalType == null) {
return PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR;
}
@@ -328,8 +328,10 @@ public final class PrimitiveType extends Type {
}
@Override
- PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
- return PrimitiveComparator.FLOAT_COMPARATOR;
+ PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
+ return columnOrder != null && columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER
+ ? PrimitiveComparator.FLOAT_IEEE_754_TOTAL_ORDER_COMPARATOR
+ : PrimitiveComparator.FLOAT_COMPARATOR;
}
},
DOUBLE("getDouble", Double.TYPE) {
@@ -354,8 +356,10 @@ public final class PrimitiveType extends Type {
}
@Override
- PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
- return PrimitiveComparator.DOUBLE_COMPARATOR;
+ PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
+ return columnOrder != null && columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER
+ ? PrimitiveComparator.DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR
+ : PrimitiveComparator.DOUBLE_COMPARATOR;
}
},
INT96("getBinary", Binary.class) {
@@ -380,7 +384,7 @@ public final class PrimitiveType extends Type {
}
@Override
- PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
+ PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
return PrimitiveComparator.BINARY_AS_SIGNED_INTEGER_COMPARATOR;
}
},
@@ -406,7 +410,7 @@ public final class PrimitiveType extends Type {
}
@Override
- PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
+ PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
if (logicalType == null) {
return PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR;
}
@@ -433,7 +437,11 @@ public final class PrimitiveType extends Type {
@Override
public Optional<PrimitiveComparator> visit(
LogicalTypeAnnotation.Float16LogicalTypeAnnotation float16LogicalType) {
- return of(PrimitiveComparator.BINARY_AS_FLOAT16_COMPARATOR);
+ return (columnOrder != null
+ && columnOrder.getColumnOrderName()
+ == ColumnOrderName.IEEE_754_TOTAL_ORDER)
+ ? of(PrimitiveComparator.BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR)
+ : of(PrimitiveComparator.BINARY_AS_FLOAT16_COMPARATOR);
}
@Override
@@ -477,7 +485,11 @@ public final class PrimitiveType extends Type {
public abstract <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> converter) throws E;
- abstract PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType);
+ abstract PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder);
+
+ public PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType) {
+ return comparator(logicalType, ColumnOrder.typeDefined());
+ }
}
private final PrimitiveTypeName primitive;
@@ -569,6 +581,12 @@ public final class PrimitiveType extends Type {
columnOrder = primitive == PrimitiveTypeName.INT96 || originalType == OriginalType.INTERVAL
? ColumnOrder.undefined()
: ColumnOrder.typeDefined();
+ } else if (columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER) {
+ Preconditions.checkArgument(
+ primitive == PrimitiveTypeName.FLOAT || primitive == PrimitiveTypeName.DOUBLE,
+ "The column order %s is not supported by type %s",
+ columnOrder,
+ primitive);
}
this.columnOrder = requireValidColumnOrder(columnOrder);
}
@@ -615,6 +633,17 @@ public final class PrimitiveType extends Type {
|| logicalTypeAnnotation instanceof LogicalTypeAnnotation.IntervalLogicalTypeAnnotation
? ColumnOrder.undefined()
: ColumnOrder.typeDefined();
+ } else if (columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER) {
+ Preconditions.checkArgument(
+ primitive == PrimitiveTypeName.FLOAT
+ || primitive == PrimitiveTypeName.DOUBLE
+ || (logicalTypeAnnotation != null
+ && logicalTypeAnnotation.getType()
+ == LogicalTypeAnnotation.LogicalTypeToken.FLOAT16),
+ "The column order %s is not supported by type %s logical type %s",
+ columnOrder,
+ primitive,
+ logicalTypeAnnotation);
}
this.columnOrder = requireValidColumnOrder(columnOrder);
}
@@ -655,6 +684,15 @@ public final class PrimitiveType extends Type {
return new PrimitiveType(getRepetition(), primitive, length, getName(), logicalType, getId());
}
+ /**
+ * @param columnOrder the column order
+ * @return the same type with the column order set
+ */
+ public Type withColumnOrder(ColumnOrder columnOrder) {
+ return new PrimitiveType(
+ getRepetition(), primitive, length, getName(), getLogicalTypeAnnotation(), getId(), columnOrder);
+ }
+
/**
* @return the primitive type
*/
@@ -856,7 +894,7 @@ public final class PrimitiveType extends Type {
builder.length(length);
}
- return builder.as(getLogicalTypeAnnotation()).named(getName());
+ return builder.as(getLogicalTypeAnnotation()).columnOrder(columnOrder()).named(getName());
}
/**
@@ -869,7 +907,7 @@ public final class PrimitiveType extends Type {
*/
@SuppressWarnings("unchecked")
public <T> PrimitiveComparator<T> comparator() {
- return (PrimitiveComparator<T>) getPrimitiveTypeName().comparator(getLogicalTypeAnnotation());
+ return (PrimitiveComparator<T>) getPrimitiveTypeName().comparator(getLogicalTypeAnnotation(), columnOrder());
}
/**
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatisticsNanCount.java b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatisticsNanCount.java
new file mode 100644
index 000000000..a1c47ada1
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/column/statistics/TestStatisticsNanCount.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.column.statistics;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.parquet.bytes.BytesUtils;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+import org.junit.Test;
+
+public class TestStatisticsNanCount {
+
+ private static final PrimitiveType FLOAT_TYPE =
+ Types.optional(PrimitiveTypeName.FLOAT).named("test_float");
+ private static final PrimitiveType DOUBLE_TYPE =
+ Types.optional(PrimitiveTypeName.DOUBLE).named("test_double");
+ private static final PrimitiveType FLOAT16_TYPE = Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(LogicalTypeAnnotation.float16Type())
+ .named("test_float16");
+
+ private static final PrimitiveType FLOAT_IEEE754_TYPE = Types.optional(PrimitiveTypeName.FLOAT)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("test_float_ieee754");
+ private static final PrimitiveType DOUBLE_IEEE754_TYPE = Types.optional(PrimitiveTypeName.DOUBLE)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("test_double_ieee754");
+ private static final PrimitiveType FLOAT16_IEEE754_TYPE = Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(LogicalTypeAnnotation.float16Type())
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("test_float16_ieee754");
+
+ private static Binary float16Binary(short h) {
+ return Binary.fromConstantByteArray(new byte[] {(byte) (h & 0xFF), (byte) ((h >> 8) & 0xFF)});
+ }
+
+ private static final Binary FLOAT16_NAN = float16Binary((short) 0x7e00);
+ private static final Binary FLOAT16_NAN_SMALL = float16Binary((short) 0x7c01);
+ private static final Binary FLOAT16_NAN_LARGE = float16Binary((short) 0x7fff);
+ private static final Binary FLOAT16_ONE = float16Binary((short) 0x3C00);
+ private static final Binary FLOAT16_TWO = float16Binary((short) 0x4000);
+
+ @Test
+ public void testFloatNanCountMixedValues() {
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(FLOAT_TYPE);
+ stats.updateStats(1.0f);
+ stats.updateStats(Float.NaN);
+ stats.updateStats(2.0f);
+ stats.updateStats(Float.NaN);
+ stats.updateStats(3.0f);
+
+ assertTrue(stats.isNanCountSet());
+ assertEquals(2, stats.getNanCount());
+ assertEquals(1.0f, stats.getMin(), 0.0f);
+ assertEquals(3.0f, stats.getMax(), 0.0f);
+ }
+
+ @Test
+ public void testFloatNanCountAllNaN() {
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(FLOAT_TYPE);
+ stats.updateStats(Float.NaN);
+ stats.updateStats(Float.NaN);
+
+ assertTrue(stats.isNanCountSet());
+ assertEquals(2, stats.getNanCount());
+ assertFalse(stats.hasNonNullValue());
+ }
+
+ @Test
+ public void testFloatNanCountNoNaN() {
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(FLOAT_TYPE);
+ stats.updateStats(1.0f);
+ stats.updateStats(2.0f);
+
+ assertTrue(stats.isNanCountSet());
+ assertEquals(0, stats.getNanCount());
+ }
+
+ @Test
+ public void testDoubleNanCountMixedValues() {
+ DoubleStatistics stats = (DoubleStatistics) Statistics.createStats(DOUBLE_TYPE);
+ stats.updateStats(1.0);
+ stats.updateStats(Double.NaN);
+ stats.updateStats(2.0);
+
+ assertTrue(stats.isNanCountSet());
+ assertEquals(1, stats.getNanCount());
+ assertEquals(1.0, stats.getMin(), 0.0);
+ assertEquals(2.0, stats.getMax(), 0.0);
+ }
+
+ @Test
+ public void testFloat16NanCountMixedValues() {
+ Float16Statistics stats = (Float16Statistics) Statistics.createStats(FLOAT16_TYPE);
+ stats.updateStats(FLOAT16_ONE);
+ stats.updateStats(FLOAT16_NAN);
+ stats.updateStats(FLOAT16_TWO);
+
+ assertTrue(stats.isNanCountSet());
+ assertEquals(1, stats.getNanCount());
+ assertTrue(stats.hasNonNullValue());
+ assertEquals(FLOAT16_ONE, stats.genericGetMin());
+ assertEquals(FLOAT16_TWO, stats.genericGetMax());
+ }
+
+ @Test
+ public void testFloat16NanCountNoNaN() {
+ Float16Statistics stats = (Float16Statistics) Statistics.createStats(FLOAT16_TYPE);
+ stats.updateStats(FLOAT16_ONE);
+
+ assertTrue(stats.isNanCountSet());
+ assertEquals(0, stats.getNanCount());
+ }
+
+ @Test
+ public void testMergeNanCounts() {
+ FloatStatistics stats1 = (FloatStatistics) Statistics.createStats(FLOAT_TYPE);
+ stats1.updateStats(1.0f);
+ stats1.updateStats(Float.NaN);
+
+ FloatStatistics stats2 = (FloatStatistics) Statistics.createStats(FLOAT_TYPE);
+ stats2.updateStats(2.0f);
+ stats2.updateStats(Float.NaN);
+ stats2.updateStats(Float.NaN);
+
+ stats1.mergeStatistics(stats2);
+ assertEquals(3, stats1.getNanCount());
+ }
+
+ @Test
+ public void testCopyPreservesNanCount() {
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(FLOAT_TYPE);
+ stats.updateStats(1.0f);
+ stats.updateStats(Float.NaN);
+ stats.updateStats(Float.NaN);
+
+ FloatStatistics copy = stats.copy();
+ assertEquals(stats.getNanCount(), copy.getNanCount());
+ assertTrue(copy.isNanCountSet());
+ assertEquals(2, copy.getNanCount());
+ }
+
+ @Test
+ public void testFloatBuilderIEEE754KeepsNanMinMax() {
+ Statistics.Builder builder = Statistics.getBuilderForReading(FLOAT_IEEE754_TYPE);
+ byte[] nanBytes = BytesUtils.intToBytes(Float.floatToIntBits(Float.NaN));
+ Statistics<?> stats = builder.withMin(nanBytes)
+ .withMax(nanBytes)
+ .withNanCount(10)
+ .withNumNulls(0)
+ .build();
+
+ assertTrue(stats.hasNonNullValue());
+ assertTrue(Float.isNaN(((FloatStatistics) stats).getMin()));
+ assertTrue(Float.isNaN(((FloatStatistics) stats).getMax()));
+ assertEquals(10, stats.getNanCount());
+ }
+
+ @Test
+ public void testFloatBuilderTypeDefinedDropsNanMinMax() {
+ Statistics.Builder builder = Statistics.getBuilderForReading(FLOAT_TYPE);
+ byte[] nanBytes = BytesUtils.intToBytes(Float.floatToIntBits(Float.NaN));
+ Statistics<?> stats =
+ builder.withMin(nanBytes).withMax(nanBytes).withNumNulls(0).build();
+
+ assertFalse(stats.hasNonNullValue());
+ assertFalse(Float.isNaN(((FloatStatistics) stats).getMin()));
+ assertFalse(Float.isNaN(((FloatStatistics) stats).getMax()));
+ }
+
+ @Test
+ public void testDoubleBuilderIEEE754KeepsNanMinMax() {
+ Statistics.Builder builder = Statistics.getBuilderForReading(DOUBLE_IEEE754_TYPE);
+ byte[] nanBytes = BytesUtils.longToBytes(Double.doubleToLongBits(Double.NaN));
+ Statistics<?> stats = builder.withMin(nanBytes)
+ .withMax(nanBytes)
+ .withNanCount(10)
+ .withNumNulls(0)
+ .build();
+
+ assertTrue(stats.hasNonNullValue());
+ assertTrue(Double.isNaN(((DoubleStatistics) stats).getMin()));
+ assertTrue(Double.isNaN(((DoubleStatistics) stats).getMax()));
+ assertEquals(10, stats.getNanCount());
+ }
+
+ @Test
+ public void testFloatIEEE754NanOnlySetsHasNonNullValue() {
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(FLOAT_IEEE754_TYPE);
+ stats.updateStats(Float.NaN);
+ stats.updateStats(Float.NaN);
+
+ assertTrue(stats.hasNonNullValue());
+ assertTrue(Float.isNaN(stats.getMin()));
+ assertTrue(Float.isNaN(stats.getMax()));
+ assertEquals(2, stats.getNanCount());
+ }
+
+ @Test
+ public void testDoubleIEEE754NanOnlySetsHasNonNullValue() {
+ DoubleStatistics stats = (DoubleStatistics) Statistics.createStats(DOUBLE_IEEE754_TYPE);
+ stats.updateStats(Double.NaN);
+
+ assertTrue(stats.hasNonNullValue());
+ assertTrue(Double.isNaN(stats.getMin()));
+ assertTrue(Double.isNaN(stats.getMax()));
+ assertEquals(1, stats.getNanCount());
+ }
+
+ @Test
+ public void testFloat16IEEE754NanOnlySetsHasNonNullValue() {
+ IEEE754Float16Statistics stats = (IEEE754Float16Statistics) Statistics.createStats(FLOAT16_IEEE754_TYPE);
+ stats.updateStats(FLOAT16_NAN);
+
+ assertTrue(stats.hasNonNullValue());
+ assertEquals(FLOAT16_NAN, stats.genericGetMin());
+ assertEquals(FLOAT16_NAN, stats.genericGetMax());
+ assertEquals(1, stats.getNanCount());
+ }
+
+ @Test
+ public void testFloatIEEE754AllNaNTracksNaNRange() {
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(FLOAT_IEEE754_TYPE);
+ float minNaN = Float.intBitsToFloat(0x7fc00001);
+ float maxNaN = Float.intBitsToFloat(0x7fffffff);
+ stats.updateStats(maxNaN);
+ stats.updateStats(minNaN);
+
+ assertEquals(2, stats.getNanCount());
+ assertEquals(0x7fc00001, Float.floatToRawIntBits(stats.getMin()));
+ assertEquals(0x7fffffff, Float.floatToRawIntBits(stats.getMax()));
+ }
+
+ @Test
+ public void testDoubleIEEE754AllNaNTracksNaNRange() {
+ DoubleStatistics stats = (DoubleStatistics) Statistics.createStats(DOUBLE_IEEE754_TYPE);
+ double minNaN = Double.longBitsToDouble(0x7ff0000000000001L);
+ double maxNaN = Double.longBitsToDouble(0x7fffffffffffffffL);
+ stats.updateStats(maxNaN);
+ stats.updateStats(minNaN);
+
+ assertEquals(2, stats.getNanCount());
+ assertEquals(0x7ff0000000000001L, Double.doubleToRawLongBits(stats.getMin()));
+ assertEquals(0x7fffffffffffffffL, Double.doubleToRawLongBits(stats.getMax()));
+ }
+
+ @Test
+ public void testFloat16IEEE754AllNaNTracksNaNRange() {
+ IEEE754Float16Statistics stats = (IEEE754Float16Statistics) Statistics.createStats(FLOAT16_IEEE754_TYPE);
+ stats.updateStats(FLOAT16_NAN_LARGE);
+ stats.updateStats(FLOAT16_NAN_SMALL);
+
+ assertEquals(2, stats.getNanCount());
+ assertEquals(
+ FLOAT16_NAN_SMALL.get2BytesLittleEndian(), stats.genericGetMin().get2BytesLittleEndian());
+ assertEquals(
+ FLOAT16_NAN_LARGE.get2BytesLittleEndian(), stats.genericGetMax().get2BytesLittleEndian());
+ }
+
+ @Test
+ public void testFloatIEEE754NonNaNReplacesAllNaNRange() {
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(FLOAT_IEEE754_TYPE);
+ stats.updateStats(Float.intBitsToFloat(0x7fffffff));
+ stats.updateStats(Float.intBitsToFloat(0x7fc00001));
+ stats.updateStats(1.0f);
+ stats.updateStats(2.0f);
+
+ assertEquals(2, stats.getNanCount());
+ assertEquals(1.0f, stats.getMin(), 0.0f);
+ assertEquals(2.0f, stats.getMax(), 0.0f);
+ }
+
+ @Test
+ public void testDoubleIEEE754NonNaNReplacesAllNaNRange() {
+ DoubleStatistics stats = (DoubleStatistics) Statistics.createStats(DOUBLE_IEEE754_TYPE);
+ stats.updateStats(Double.longBitsToDouble(0x7fffffffffffffffL));
+ stats.updateStats(Double.longBitsToDouble(0x7ff0000000000001L));
+ stats.updateStats(1.0);
+ stats.updateStats(2.0);
+
+ assertEquals(2, stats.getNanCount());
+ assertEquals(1.0, stats.getMin(), 0.0);
+ assertEquals(2.0, stats.getMax(), 0.0);
+ }
+
+ @Test
+ public void testFloat16IEEE754NonNaNReplacesAllNaNRange() {
+ IEEE754Float16Statistics stats = (IEEE754Float16Statistics) Statistics.createStats(FLOAT16_IEEE754_TYPE);
+ stats.updateStats(FLOAT16_NAN_LARGE);
+ stats.updateStats(FLOAT16_NAN_SMALL);
+ stats.updateStats(FLOAT16_ONE);
+ stats.updateStats(FLOAT16_TWO);
+
+ assertEquals(2, stats.getNanCount());
+ assertEquals(FLOAT16_ONE, stats.genericGetMin());
+ assertEquals(FLOAT16_TWO, stats.genericGetMax());
+ }
+
+ @Test
+ public void testFloatIEEE754NanExcludedFromMax() {
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(FLOAT_IEEE754_TYPE);
+ stats.updateStats(1.0f);
+ stats.updateStats(Float.NaN);
+ stats.updateStats(2.0f);
+
+ assertEquals(2.0f, stats.getMax(), 0.0f);
+ assertEquals(1.0f, stats.getMin(), 0.0f);
+ assertEquals(1, stats.getNanCount());
+ }
+
+ @Test
+ public void testDoubleIEEE754NanExcludedFromMax() {
+ DoubleStatistics stats = (DoubleStatistics) Statistics.createStats(DOUBLE_IEEE754_TYPE);
+ stats.updateStats(1.0);
+ stats.updateStats(Double.NaN);
+ stats.updateStats(2.0);
+
+ assertEquals(2.0, stats.getMax(), 0.0);
+ assertEquals(1.0, stats.getMin(), 0.0);
+ assertEquals(1, stats.getNanCount());
+ }
+
+ @Test
+ public void testFloatTypeDefinedNanOnlyDoesNotSetHasNonNullValue() {
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(FLOAT_TYPE);
+ stats.updateStats(Float.NaN);
+ stats.updateStats(Float.NaN);
+
+ assertFalse(stats.hasNonNullValue());
+ assertEquals(2, stats.getNanCount());
+ }
+
+ @Test
+ public void testDoubleTypeDefinedNanOnlyDoesNotSetHasNonNullValue() {
+ DoubleStatistics stats = (DoubleStatistics) Statistics.createStats(DOUBLE_TYPE);
+ stats.updateStats(Double.NaN);
+
+ assertFalse(stats.hasNonNullValue());
+ assertEquals(1, stats.getNanCount());
+ }
+
+ @Test
+ public void testFloat16TypeDefinedNanOnlyDoesNotSetHasNonNullValue() {
+ Float16Statistics stats = (Float16Statistics) Statistics.createStats(FLOAT16_TYPE);
+ stats.updateStats(FLOAT16_NAN);
+
+ assertFalse(stats.hasNonNullValue());
+ assertEquals(1, stats.getNanCount());
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesEndToEndTest.java b/parquet-column/src/test/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesEndToEndTest.java
index 259b062b9..cef7f0ba7 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesEndToEndTest.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/bytestreamsplit/ByteStreamSplitValuesEndToEndTest.java
@@ -69,6 +69,40 @@ public class ByteStreamSplitValuesEndToEndTest {
}
}
+ @Test
+ public void testFloatPipelinePreservesNaNBits() throws Exception {
+ float[] values = {
+ Float.intBitsToFloat(0xffffffff),
+ Float.intBitsToFloat(0xfff00001),
+ -0.0f,
+ 0.0f,
+ Float.intBitsToFloat(0x7fc00001),
+ Float.intBitsToFloat(0x7fffffff)
+ };
+
+ ByteStreamSplitValuesWriter.FloatByteStreamSplitValuesWriter writer = null;
+ try {
+ writer = new ByteStreamSplitValuesWriter.FloatByteStreamSplitValuesWriter(
+ values.length * 4, values.length * 4, new DirectByteBufferAllocator());
+ for (float value : values) {
+ writer.writeFloat(value);
+ }
+
+ BytesInput input = writer.getBytes();
+ ByteStreamSplitValuesReaderForFloat reader = new ByteStreamSplitValuesReaderForFloat();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (float expectedValue : values) {
+ assertEquals(Float.floatToRawIntBits(expectedValue), Float.floatToRawIntBits(reader.readFloat()));
+ }
+ } finally {
+ if (writer != null) {
+ writer.reset();
+ writer.close();
+ }
+ }
+ }
+
@Test
public void testDoublePipeline() throws Exception {
// Generate random data.
@@ -101,6 +135,36 @@ public class ByteStreamSplitValuesEndToEndTest {
writer.close();
}
+ @Test
+ public void testDoublePipelinePreservesNaNBits() throws Exception {
+ double[] values = {
+ Double.longBitsToDouble(0xffffffffffffffffL),
+ Double.longBitsToDouble(0xfff0000000000001L),
+ -0.0d,
+ 0.0d,
+ Double.longBitsToDouble(0x7ff0000000000001L),
+ Double.longBitsToDouble(0x7fffffffffffffffL)
+ };
+
+ ByteStreamSplitValuesWriter.DoubleByteStreamSplitValuesWriter writer =
+ new ByteStreamSplitValuesWriter.DoubleByteStreamSplitValuesWriter(
+ values.length * 8, values.length * 8, new DirectByteBufferAllocator());
+ for (double value : values) {
+ writer.writeDouble(value);
+ }
+
+ BytesInput input = writer.getBytes();
+ ByteStreamSplitValuesReaderForDouble reader = new ByteStreamSplitValuesReaderForDouble();
+ reader.initFromPage(values.length, ByteBufferInputStream.wrap(input.toByteBuffer()));
+
+ for (double expectedValue : values) {
+ assertEquals(Double.doubleToRawLongBits(expectedValue), Double.doubleToRawLongBits(reader.readDouble()));
+ }
+
+ writer.reset();
+ writer.close();
+ }
+
@Test
public void testIntegerPipeline() throws Exception {
// Generate random data.
diff --git a/parquet-column/src/test/java/org/apache/parquet/column/values/dictionary/TestDictionary.java b/parquet-column/src/test/java/org/apache/parquet/column/values/dictionary/TestDictionary.java
index a91f807e7..c7cf35199 100644
--- a/parquet-column/src/test/java/org/apache/parquet/column/values/dictionary/TestDictionary.java
+++ b/parquet-column/src/test/java/org/apache/parquet/column/values/dictionary/TestDictionary.java
@@ -413,6 +413,37 @@ public class TestDictionary {
}
}
+ @Test
+ public void testDoubleDictionaryPreservesNaNBits() throws IOException {
+ double[] dictionaryValues = {
+ Double.longBitsToDouble(0xffffffffffffffffL),
+ Double.longBitsToDouble(0xfff0000000000001L),
+ -0.0d,
+ 0.0d,
+ Double.longBitsToDouble(0x7ff0000000000001L),
+ Double.longBitsToDouble(0x7fffffffffffffffL)
+ };
+ try (final FallbackValuesWriter<PlainDoubleDictionaryValuesWriter, PlainValuesWriter> cw =
+ newPlainDoubleDictionaryValuesWriter(10000, 10000)) {
+ for (int i = 0; i < 10; i++) {
+ for (double value : dictionaryValues) {
+ cw.writeDouble(value);
+ }
+ }
+
+ BytesInput bytes = getBytesAndCheckEncoding(cw, PLAIN_DICTIONARY);
+ assertEquals(dictionaryValues.length, cw.initialWriter.getDictionarySize());
+
+ final DictionaryValuesReader cr = initDicReader(cw, DOUBLE);
+ cr.initFromPage(dictionaryValues.length * 10, bytes.toInputStream());
+ for (int i = 0; i < 10; i++) {
+ for (double expected : dictionaryValues) {
+ assertEquals(Double.doubleToRawLongBits(expected), Double.doubleToRawLongBits(cr.readDouble()));
+ }
+ }
+ }
+ }
+
private void roundTripDouble(
FallbackValuesWriter<PlainDoubleDictionaryValuesWriter, PlainValuesWriter> cw,
ValuesReader reader,
@@ -601,6 +632,37 @@ public class TestDictionary {
}
}
+ @Test
+ public void testFloatDictionaryPreservesNaNBits() throws IOException {
+ float[] dictionaryValues = {
+ Float.intBitsToFloat(0xffffffff),
+ Float.intBitsToFloat(0xfff00001),
+ -0.0f,
+ 0.0f,
+ Float.intBitsToFloat(0x7fc00001),
+ Float.intBitsToFloat(0x7fffffff)
+ };
+ try (final FallbackValuesWriter<PlainFloatDictionaryValuesWriter, PlainValuesWriter> cw =
+ newPlainFloatDictionaryValuesWriter(10000, 10000)) {
+ for (int i = 0; i < 10; i++) {
+ for (float value : dictionaryValues) {
+ cw.writeFloat(value);
+ }
+ }
+
+ BytesInput bytes = getBytesAndCheckEncoding(cw, PLAIN_DICTIONARY);
+ assertEquals(dictionaryValues.length, cw.initialWriter.getDictionarySize());
+
+ final DictionaryValuesReader cr = initDicReader(cw, FLOAT);
+ cr.initFromPage(dictionaryValues.length * 10, bytes.toInputStream());
+ for (int i = 0; i < 10; i++) {
+ for (float expected : dictionaryValues) {
+ assertEquals(Float.floatToRawIntBits(expected), Float.floatToRawIntBits(cr.readFloat()));
+ }
+ }
+ }
+ }
+
private void roundTripFloat(
FallbackValuesWriter<PlainFloatDictionaryValuesWriter, PlainValuesWriter> cw,
ValuesReader reader,
diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilderNaN.java b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilderNaN.java
new file mode 100644
index 000000000..85ef5a9b1
--- /dev/null
+++ b/parquet-column/src/test/java/org/apache/parquet/internal/column/columnindex/TestColumnIndexBuilderNaN.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.internal.column.columnindex;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.PrimitiveIterator;
+import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.filter2.predicate.FilterApi;
+import org.apache.parquet.filter2.predicate.Operators.BinaryColumn;
+import org.apache.parquet.filter2.predicate.Operators.DoubleColumn;
+import org.apache.parquet.filter2.predicate.Operators.FloatColumn;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+import org.junit.Test;
+
+/**
+ * Tests for ColumnIndex NaN handling under IEEE_754_TOTAL_ORDER and TYPE_DEFINED_ORDER.
+ */
+public class TestColumnIndexBuilderNaN {
+
+ private static final PrimitiveType FLOAT_TYPE =
+ Types.required(PrimitiveTypeName.FLOAT).named("test_float");
+ private static final PrimitiveType FLOAT_IEEE754_TYPE = Types.required(PrimitiveTypeName.FLOAT)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("test_float_ieee754");
+ private static final PrimitiveType DOUBLE_TYPE =
+ Types.required(PrimitiveTypeName.DOUBLE).named("test_double");
+ private static final PrimitiveType DOUBLE_IEEE754_TYPE = Types.required(PrimitiveTypeName.DOUBLE)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("test_double_ieee754");
+ private static final PrimitiveType FLOAT16_TYPE = Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(LogicalTypeAnnotation.float16Type())
+ .named("test_float16");
+ private static final PrimitiveType FLOAT16_IEEE754_TYPE = Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(LogicalTypeAnnotation.float16Type())
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("test_float16_ieee754");
+
+ private static final FloatColumn FLOAT_COL = FilterApi.floatColumn("test_float_ieee754");
+ private static final DoubleColumn DOUBLE_COL = FilterApi.doubleColumn("test_double_ieee754");
+ private static final BinaryColumn FLOAT16_COL = FilterApi.binaryColumn("test_float16_ieee754");
+
+ private static Binary float16Binary(short h) {
+ return Binary.fromConstantByteArray(new byte[] {(byte) (h & 0xFF), (byte) ((h >> 8) & 0xFF)});
+ }
+
+ private static final Binary FLOAT16_NAN = float16Binary((short) 0x7e00);
+ private static final Binary FLOAT16_NAN_SMALL = float16Binary((short) 0x7c01);
+ private static final Binary FLOAT16_NAN_LARGE = float16Binary((short) 0x7fff);
+ private static final Binary FLOAT16_ONE = float16Binary((short) 0x3C00); // 1.0
+ private static final Binary FLOAT16_TWO = float16Binary((short) 0x4000); // 2.0
+ private static final Binary FLOAT16_THREE = float16Binary((short) 0x4200); // 3.0
+ private static final Binary FLOAT16_FOUR = float16Binary((short) 0x4400); // 4.0
+
+ private static Statistics<?> floatStats(PrimitiveType type, float... values) {
+ Statistics<?> stats = Statistics.createStats(type);
+ for (float value : values) {
+ stats.updateStats(value);
+ }
+ return stats;
+ }
+
+ private static Statistics<?> doubleStats(PrimitiveType type, double... values) {
+ Statistics<?> stats = Statistics.createStats(type);
+ for (double value : values) {
+ stats.updateStats(value);
+ }
+ return stats;
+ }
+
+ private static Statistics<?> binaryStats(PrimitiveType type, Binary... values) {
+ Statistics<?> stats = Statistics.createStats(type);
+ for (Binary value : values) {
+ stats.updateStats(value);
+ }
+ return stats;
+ }
+
+ private static List<Integer> toList(PrimitiveIterator.OfInt iter) {
+ List<Integer> result = new ArrayList<>();
+ iter.forEachRemaining((int i) -> result.add(i));
+ return result;
+ }
+
+ private static ByteBuffer floatBuffer(float value) {
+ ByteBuffer buffer = ByteBuffer.allocate(Float.BYTES).order(ByteOrder.LITTLE_ENDIAN);
+ buffer.putFloat(0, value);
+ return buffer;
+ }
+
+ // TYPE_DEFINED_ORDER: build column index with NaN
+
+ @Test
+ public void testFloatTypeDefinedOrderNaN() {
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT_TYPE, Integer.MAX_VALUE);
+ builder.add(floatStats(FLOAT_TYPE, 1.0f, 2.0f));
+ builder.add(floatStats(FLOAT_TYPE, Float.NaN));
+ builder.add(floatStats(FLOAT_TYPE, 3.0f, 4.0f));
+ assertNull(builder.build());
+ }
+
+ @Test
+ public void testDoubleTypeDefinedOrderNaN() {
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(DOUBLE_TYPE, Integer.MAX_VALUE);
+ builder.add(doubleStats(DOUBLE_TYPE, 1.0, 2.0));
+ builder.add(doubleStats(DOUBLE_TYPE, Double.NaN));
+ builder.add(doubleStats(DOUBLE_TYPE, 3.0, 4.0));
+ assertNull(builder.build());
+ }
+
+ @Test
+ public void testFloat16TypeDefinedOrderNaN() {
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT16_TYPE, Integer.MAX_VALUE);
+ builder.add(binaryStats(FLOAT16_TYPE, FLOAT16_ONE, FLOAT16_TWO));
+ builder.add(binaryStats(FLOAT16_TYPE, FLOAT16_NAN));
+ builder.add(binaryStats(FLOAT16_TYPE, FLOAT16_ONE));
+ assertNull(builder.build());
+ }
+
+ // IEEE_754_TOTAL_ORDER: build column index with NaN
+
+ @Test
+ public void testFloatIeee754BuildNanCounts() {
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(floatStats(FLOAT_IEEE754_TYPE, 1.0f, 2.0f));
+ builder.add(floatStats(FLOAT_IEEE754_TYPE, Float.NaN, 2.5f, Float.NaN));
+ builder.add(floatStats(FLOAT_IEEE754_TYPE, 3.0f, Float.NaN, 4.0f));
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+ assertEquals(List.of(0L, 2L, 1L), ci.getNanCounts());
+ }
+
+ @Test
+ public void testDoubleIeee754BuildNanCounts() {
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(DOUBLE_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(doubleStats(DOUBLE_IEEE754_TYPE, 1.0, 2.0));
+ builder.add(doubleStats(DOUBLE_IEEE754_TYPE, Double.NaN, 2.5, Double.NaN));
+ builder.add(doubleStats(DOUBLE_IEEE754_TYPE, 3.0, Double.NaN, 4.0));
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+ assertEquals(List.of(0L, 2L, 1L), ci.getNanCounts());
+ }
+
+ @Test
+ public void testFloat16Ieee754BuildNanCounts() {
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT16_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(binaryStats(FLOAT16_IEEE754_TYPE, FLOAT16_ONE, FLOAT16_NAN, FLOAT16_TWO));
+ builder.add(binaryStats(FLOAT16_IEEE754_TYPE, FLOAT16_NAN, FLOAT16_TWO, FLOAT16_NAN));
+ builder.add(binaryStats(FLOAT16_IEEE754_TYPE, FLOAT16_ONE));
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+ assertEquals(List.of(1L, 2L, 0L), ci.getNanCounts());
+ }
+
+ @Test
+ public void testFloatIeee754AllNanPageKeepsNanRangeInColumnIndex() {
+ float minNaN = Float.intBitsToFloat(0x7fc00001);
+ float maxNaN = Float.intBitsToFloat(0x7fffffff);
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(floatStats(FLOAT_IEEE754_TYPE, maxNaN, minNaN));
+
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+ assertEquals(1, ci.getMinValues().size());
+ assertEquals(
+ 0x7fc00001,
+ ci.getMinValues().get(0).order(ByteOrder.LITTLE_ENDIAN).getInt(0));
+ assertEquals(
+ 0x7fffffff,
+ ci.getMaxValues().get(0).order(ByteOrder.LITTLE_ENDIAN).getInt(0));
+ }
+
+ @Test
+ public void testDoubleIeee754AllNanPageKeepsNanRangeInColumnIndex() {
+ double minNaN = Double.longBitsToDouble(0x7ff0000000000001L);
+ double maxNaN = Double.longBitsToDouble(0x7fffffffffffffffL);
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(DOUBLE_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(doubleStats(DOUBLE_IEEE754_TYPE, maxNaN, minNaN));
+
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+ assertEquals(1, ci.getMinValues().size());
+ assertEquals(
+ 0x7ff0000000000001L,
+ ci.getMinValues().get(0).order(ByteOrder.LITTLE_ENDIAN).getLong(0));
+ assertEquals(
+ 0x7fffffffffffffffL,
+ ci.getMaxValues().get(0).order(ByteOrder.LITTLE_ENDIAN).getLong(0));
+ }
+
+ @Test
+ public void testFloat16Ieee754AllNanPageKeepsNanRangeInColumnIndex() {
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT16_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(binaryStats(FLOAT16_IEEE754_TYPE, FLOAT16_NAN_LARGE, FLOAT16_NAN_SMALL));
+
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+ assertEquals(1, ci.getMinValues().size());
+ assertEquals(
+ FLOAT16_NAN_SMALL.get2BytesLittleEndian(),
+ ci.getMinValues().get(0).order(ByteOrder.LITTLE_ENDIAN).getShort(0));
+ assertEquals(
+ FLOAT16_NAN_LARGE.get2BytesLittleEndian(),
+ ci.getMaxValues().get(0).order(ByteOrder.LITTLE_ENDIAN).getShort(0));
+ }
+
+ // Column index filtering for float
+
+ @Test
+ public void testNaNFloatZeroNaN() {
+ // Pages: [1.0, 2.0], [3.0, 4.0]
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(floatStats(FLOAT_IEEE754_TYPE, 1.0f, 2.0f));
+ builder.add(floatStats(FLOAT_IEEE754_TYPE, 3.0f, 4.0f));
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+
+ // Non-NaN literal (1.5 within page 0 range; ASCENDING boundary order)
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.eq(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.notEq(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.lt(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.ltEq(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gt(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gtEq(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.in(FLOAT_COL, new HashSet<>(List.of(1.5f))))));
+
+ // NaN literal: nanCounts all zero, so eq returns empty and notEq returns all
+ assertEquals(List.of(), toList(ci.visit(FilterApi.eq(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.notEq(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.lt(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.ltEq(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gt(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gtEq(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(), toList(ci.visit(FilterApi.in(FLOAT_COL, new HashSet<>(List.of(Float.NaN))))));
+ }
+
+ @Test
+ public void testNaNFloatUnknownNaNCounts() {
+ ColumnIndex ci = ColumnIndexBuilder.build(
+ FLOAT_IEEE754_TYPE,
+ BoundaryOrder.ASCENDING,
+ List.of(false, false),
+ List.of(0L, 0L),
+ null,
+ List.of(floatBuffer(1.0f), floatBuffer(3.0f)),
+ List.of(floatBuffer(2.0f), floatBuffer(4.0f)),
+ null,
+ null);
+ assertNotNull(ci);
+
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.eq(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.lt(FLOAT_COL, 0.0f))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gt(FLOAT_COL, 5.0f))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.notEq(FLOAT_COL, 1.5f))));
+ }
+
+ @Test
+ public void testNaNFloatMixed() {
+ // Pages: [1.0, 2.0], [NaN, 2.5], [3.0, 4.0]
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(floatStats(FLOAT_IEEE754_TYPE, 1.0f, 2.0f));
+ builder.add(floatStats(FLOAT_IEEE754_TYPE, Float.NaN, 2.5f));
+ builder.add(floatStats(FLOAT_IEEE754_TYPE, 3.0f, 4.0f));
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.eq(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.notEq(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.lt(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.ltEq(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gt(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gtEq(FLOAT_COL, 1.5f))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.in(FLOAT_COL, new HashSet<>(List.of(1.5f))))));
+
+ assertEquals(List.of(1), toList(ci.visit(FilterApi.eq(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.notEq(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.lt(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.ltEq(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gt(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gtEq(FLOAT_COL, Float.NaN))));
+ assertEquals(List.of(1), toList(ci.visit(FilterApi.in(FLOAT_COL, new HashSet<>(List.of(Float.NaN))))));
+ }
+
+ // Column index filtering for double
+
+ @Test
+ public void testNaNDoubleZeroNaN() {
+ // Pages: [1.0, 2.0], [3.0, 4.0]
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(DOUBLE_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(doubleStats(DOUBLE_IEEE754_TYPE, 1.0, 2.0));
+ builder.add(doubleStats(DOUBLE_IEEE754_TYPE, 3.0, 4.0));
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.eq(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.notEq(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.lt(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.ltEq(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gt(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gtEq(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.in(DOUBLE_COL, new HashSet<>(List.of(1.5))))));
+
+ assertEquals(List.of(), toList(ci.visit(FilterApi.eq(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.notEq(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.lt(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.ltEq(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gt(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gtEq(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(), toList(ci.visit(FilterApi.in(DOUBLE_COL, new HashSet<>(List.of(Double.NaN))))));
+ }
+
+ @Test
+ public void testNaNDoubleMixed() {
+ // Pages: [1.0, 2.0], [NaN, 2.5], [3.0, 4.0]
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(DOUBLE_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(doubleStats(DOUBLE_IEEE754_TYPE, 1.0, 2.0));
+ builder.add(doubleStats(DOUBLE_IEEE754_TYPE, Double.NaN, 2.5));
+ builder.add(doubleStats(DOUBLE_IEEE754_TYPE, 3.0, 4.0));
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.eq(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.notEq(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.lt(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.ltEq(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gt(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gtEq(DOUBLE_COL, 1.5))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.in(DOUBLE_COL, new HashSet<>(List.of(1.5))))));
+
+ assertEquals(List.of(1), toList(ci.visit(FilterApi.eq(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.notEq(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.lt(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.ltEq(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gt(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gtEq(DOUBLE_COL, Double.NaN))));
+ assertEquals(List.of(1), toList(ci.visit(FilterApi.in(DOUBLE_COL, new HashSet<>(List.of(Double.NaN))))));
+ }
+
+ // Column index filtering for float16
+
+ @Test
+ public void testNaNFloat16ZeroNaN() {
+ // Pages: [1.0, 2.0], [3.0, 4.0]
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT16_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(binaryStats(FLOAT16_IEEE754_TYPE, FLOAT16_ONE, FLOAT16_TWO));
+ builder.add(binaryStats(FLOAT16_IEEE754_TYPE, FLOAT16_THREE, FLOAT16_FOUR));
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.eq(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.notEq(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(), toList(ci.visit(FilterApi.lt(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.ltEq(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gt(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gtEq(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.in(FLOAT16_COL, new HashSet<>(List.of(FLOAT16_ONE))))));
+
+ assertEquals(List.of(), toList(ci.visit(FilterApi.eq(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.notEq(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.lt(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.ltEq(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gt(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.gtEq(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(), toList(ci.visit(FilterApi.in(FLOAT16_COL, new HashSet<>(List.of(FLOAT16_NAN))))));
+ }
+
+ @Test
+ public void testNaNFloat16Mixed() {
+ // Pages: [1.0, 2.0], [NaN, 2.0], [3.0, 4.0]
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(FLOAT16_IEEE754_TYPE, Integer.MAX_VALUE);
+ builder.add(binaryStats(FLOAT16_IEEE754_TYPE, FLOAT16_ONE, FLOAT16_TWO));
+ builder.add(binaryStats(FLOAT16_IEEE754_TYPE, FLOAT16_NAN, FLOAT16_TWO));
+ builder.add(binaryStats(FLOAT16_IEEE754_TYPE, FLOAT16_THREE, FLOAT16_FOUR));
+ ColumnIndex ci = builder.build();
+ assertNotNull(ci);
+
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.eq(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.notEq(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(1), toList(ci.visit(FilterApi.lt(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0, 1), toList(ci.visit(FilterApi.ltEq(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gt(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gtEq(FLOAT16_COL, FLOAT16_ONE))));
+ assertEquals(List.of(0), toList(ci.visit(FilterApi.in(FLOAT16_COL, new HashSet<>(List.of(FLOAT16_ONE))))));
+
+ assertEquals(List.of(1), toList(ci.visit(FilterApi.eq(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.notEq(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.lt(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.ltEq(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gt(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(0, 1, 2), toList(ci.visit(FilterApi.gtEq(FLOAT16_COL, FLOAT16_NAN))));
+ assertEquals(List.of(1), toList(ci.visit(FilterApi.in(FLOAT16_COL, new HashSet<>(List.of(FLOAT16_NAN))))));
+ }
+}
diff --git a/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestColumnIndexFilter.java b/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestColumnIndexFilter.java
index 1574ce247..cf91ef4d7 100644
--- a/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestColumnIndexFilter.java
+++ b/parquet-column/src/test/java/org/apache/parquet/internal/filter2/columnindex/TestColumnIndexFilter.java
@@ -74,6 +74,7 @@ import org.apache.parquet.internal.column.columnindex.TestColumnIndexBuilder.Bin
import org.apache.parquet.internal.column.columnindex.TestColumnIndexBuilder.DoubleIsInteger;
import org.apache.parquet.internal.column.columnindex.TestColumnIndexBuilder.IntegerIsDivisableWith3;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.PrimitiveType;
import org.junit.Test;
@@ -87,6 +88,7 @@ public class TestColumnIndexFilter {
private final BoundaryOrder order;
private List<Boolean> nullPages = new ArrayList<>();
private List<Long> nullCounts = new ArrayList<>();
+ private List<Long> nanCounts;
private List<ByteBuffer> minValues = new ArrayList<>();
private List<ByteBuffer> maxValues = new ArrayList<>();
@@ -98,6 +100,7 @@ public class TestColumnIndexFilter {
CIBuilder addNullPage(long nullCount) {
nullPages.add(true);
nullCounts.add(nullCount);
+ addNanCount(0L);
minValues.add(EMPTY);
maxValues.add(EMPTY);
return this;
@@ -106,6 +109,7 @@ public class TestColumnIndexFilter {
CIBuilder addPage(long nullCount, int min, int max) {
nullPages.add(false);
nullCounts.add(nullCount);
+ addNanCount(0L);
minValues.add(ByteBuffer.wrap(BytesUtils.intToBytes(min)));
maxValues.add(ByteBuffer.wrap(BytesUtils.intToBytes(max)));
return this;
@@ -114,6 +118,7 @@ public class TestColumnIndexFilter {
CIBuilder addPage(long nullCount, String min, String max) {
nullPages.add(false);
nullCounts.add(nullCount);
+ addNanCount(0L);
minValues.add(ByteBuffer.wrap(min.getBytes(UTF_8)));
maxValues.add(ByteBuffer.wrap(max.getBytes(UTF_8)));
return this;
@@ -122,13 +127,37 @@ public class TestColumnIndexFilter {
CIBuilder addPage(long nullCount, double min, double max) {
nullPages.add(false);
nullCounts.add(nullCount);
+ addNanCount(0L);
minValues.add(ByteBuffer.wrap(BytesUtils.longToBytes(Double.doubleToLongBits(min))));
maxValues.add(ByteBuffer.wrap(BytesUtils.longToBytes(Double.doubleToLongBits(max))));
return this;
}
+ private void addNanCount(long nanCount) {
+ if (hasFloatingPointNanSemantics()) {
+ if (nanCounts == null) {
+ nanCounts = new ArrayList<>();
+ }
+ nanCounts.add(nanCount);
+ }
+ }
+
+ private boolean hasFloatingPointNanSemantics() {
+ switch (type.getPrimitiveTypeName()) {
+ case FLOAT:
+ case DOUBLE:
+ return true;
+ case FIXED_LEN_BYTE_ARRAY:
+ return type.getLogicalTypeAnnotation()
+ instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
+ default:
+ return false;
+ }
+ }
+
ColumnIndex build() {
- return ColumnIndexBuilder.build(type, order, nullPages, nullCounts, minValues, maxValues);
+ return ColumnIndexBuilder.build(
+ type, order, nullPages, nullCounts, nanCounts, minValues, maxValues, null, null);
}
}
diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestMessageType.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestMessageType.java
index e92bc7b3c..ac4d099ee 100644
--- a/parquet-column/src/test/java/org/apache/parquet/schema/TestMessageType.java
+++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestMessageType.java
@@ -205,7 +205,9 @@ public class TestMessageType {
Types.buildMessage()
.addFields(
Types.requiredList()
- .element(Types.optional(BINARY).named("a"))
+ .element(Types.optional(BINARY)
+ .columnOrder(ColumnOrder.undefined())
+ .named("a"))
.named("g"),
Types.optional(INT96).named("b"),
Types.optional(BINARY).named("c"))
diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java
index 8fb53aca0..ec7425141 100644
--- a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java
+++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java
@@ -19,10 +19,13 @@
package org.apache.parquet.schema;
import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_FLOAT16_COMPARATOR;
+import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_SIGNED_INTEGER_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.BOOLEAN_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.DOUBLE_COMPARATOR;
+import static org.apache.parquet.schema.PrimitiveComparator.DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.FLOAT_COMPARATOR;
+import static org.apache.parquet.schema.PrimitiveComparator.FLOAT_IEEE_754_TOTAL_ORDER_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.SIGNED_INT32_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.SIGNED_INT64_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.UNSIGNED_INT32_COMPARATOR;
@@ -167,6 +170,22 @@ public class TestPrimitiveComparator {
checkThrowingUnsupportedException(comparator, Long.TYPE);
}
+ private void testFloatComparator(PrimitiveComparator<Float> comparator, Float... valuesInAscendingOrder) {
+ for (int i = 0; i < valuesInAscendingOrder.length; ++i) {
+ for (int j = 0; j < valuesInAscendingOrder.length; ++j) {
+ Float vi = valuesInAscendingOrder[i];
+ Float vj = valuesInAscendingOrder[j];
+ int exp = i - j;
+ assertSignumEquals(vi, vj, exp, comparator.compare(vi, vj));
+ if (vi != null && vj != null) {
+ assertSignumEquals(vi, vj, exp, comparator.compare(vi.floatValue(), vj.floatValue()));
+ }
+ }
+ }
+
+ checkThrowingUnsupportedException(comparator, Float.TYPE);
+ }
+
@Test
public void testFloatComparator() {
Float[] valuesInAscendingOrder = {
@@ -182,19 +201,46 @@ public class TestPrimitiveComparator {
Float.POSITIVE_INFINITY
};
+ testFloatComparator(FLOAT_COMPARATOR, valuesInAscendingOrder);
+ }
+
+ @Test
+ public void testFloatIEEE754TotalOrderComparator() {
+ Float[] valuesInAscendingOrder = {
+ null,
+ Float.intBitsToFloat(0xFFFFFFFF), // -NaN (smallest)
+ Float.intBitsToFloat(0xFFF00001), // -NaN (largest)
+ Float.NEGATIVE_INFINITY,
+ -Float.MAX_VALUE,
+ -1234.5678F,
+ -Float.MIN_VALUE,
+ -0.0F,
+ 0.0F,
+ Float.MIN_VALUE,
+ 1234.5678F,
+ Float.MAX_VALUE,
+ Float.POSITIVE_INFINITY,
+ Float.intBitsToFloat(0x7FF00001), // +NaN (smallest)
+ Float.intBitsToFloat(0x7FFFFFFF), // +NaN (largest)
+ };
+
+ testFloatComparator(FLOAT_IEEE_754_TOTAL_ORDER_COMPARATOR, valuesInAscendingOrder);
+ }
+
+ private void testDoubleComparator(PrimitiveComparator<Double> comparator, Double... valuesInAscendingOrder) {
for (int i = 0; i < valuesInAscendingOrder.length; ++i) {
for (int j = 0; j < valuesInAscendingOrder.length; ++j) {
- Float vi = valuesInAscendingOrder[i];
- Float vj = valuesInAscendingOrder[j];
+ Double vi = valuesInAscendingOrder[i];
+ Double vj = valuesInAscendingOrder[j];
int exp = i - j;
- assertSignumEquals(vi, vj, exp, FLOAT_COMPARATOR.compare(vi, vj));
+ assertSignumEquals(vi, vj, exp, comparator.compare(vi, vj));
if (vi != null && vj != null) {
- assertSignumEquals(vi, vj, exp, FLOAT_COMPARATOR.compare(vi.floatValue(), vj.floatValue()));
+ assertSignumEquals(vi, vj, exp, comparator.compare(vi.doubleValue(), vj.doubleValue()));
}
}
}
- checkThrowingUnsupportedException(FLOAT_COMPARATOR, Float.TYPE);
+ checkThrowingUnsupportedException(comparator, Double.TYPE);
}
@Test
@@ -212,19 +258,30 @@ public class TestPrimitiveComparator {
Double.POSITIVE_INFINITY
};
- for (int i = 0; i < valuesInAscendingOrder.length; ++i) {
- for (int j = 0; j < valuesInAscendingOrder.length; ++j) {
- Double vi = valuesInAscendingOrder[i];
- Double vj = valuesInAscendingOrder[j];
- int exp = i - j;
- assertSignumEquals(vi, vj, exp, DOUBLE_COMPARATOR.compare(vi, vj));
- if (vi != null && vj != null) {
- assertSignumEquals(vi, vj, exp, DOUBLE_COMPARATOR.compare(vi.doubleValue(), vj.doubleValue()));
- }
- }
- }
+ testDoubleComparator(DOUBLE_COMPARATOR, valuesInAscendingOrder);
+ }
- checkThrowingUnsupportedException(DOUBLE_COMPARATOR, Double.TYPE);
+ @Test
+ public void testDoubleIEEE754TotalOrderComparator() {
+ Double[] valuesInAscendingOrder = {
+ null,
+ Double.longBitsToDouble(0xFFFFFFFFFFFFFFFFL), // -NaN (smallest)
+ Double.longBitsToDouble(0xFFF0000000000001L), // -NaN (largest)
+ Double.NEGATIVE_INFINITY,
+ -Double.MAX_VALUE,
+ -123456.7890123456789,
+ -Double.MIN_VALUE,
+ -0.0,
+ +0.0,
+ Double.MIN_VALUE,
+ 123456.7890123456789,
+ Double.MAX_VALUE,
+ Double.POSITIVE_INFINITY,
+ Double.longBitsToDouble(0x7FF0000000000001L), // +NaN (smallest)
+ Double.longBitsToDouble(0x7FFFFFFFFFFFFFFFL), // +NaN (largest)
+ };
+
+ testDoubleComparator(DOUBLE_IEEE_754_TOTAL_ORDER_COMPARATOR, valuesInAscendingOrder);
}
@Test
@@ -324,6 +381,34 @@ public class TestPrimitiveComparator {
}
}
+ @Test
+ public void testBinaryAsFloat16IEEE754TotalOrderComparator() {
+ Binary[] valuesInAscendingOrder = {
+ null,
+ Binary.fromConstantByteArray(new byte[] {(byte) 0xff, (byte) 0xff}), // -NaN (smallest)
+ Binary.fromConstantByteArray(new byte[] {(byte) 0x01, (byte) 0xfc}), // -NaN (largest)
+ Binary.fromConstantByteArray(new byte[] {0x00, (byte) 0xfc}), // -Infinity
+ Binary.fromConstantByteArray(new byte[] {0x00, (byte) 0xc0}), // -2.0
+ Binary.fromConstantByteArray(new byte[] {(byte) 0x01, (byte) 0x84}), // -6.109476E-5
+ Binary.fromConstantByteArray(new byte[] {0x00, (byte) 0x80}), // -0
+ Binary.fromConstantByteArray(new byte[] {0x00, 0x00}), // +0
+ Binary.fromConstantByteArray(new byte[] {(byte) 0x01, (byte) 0x00}), // 5.9604645E-8
+ Binary.fromConstantByteArray(new byte[] {(byte) 0xff, (byte) 0x7b}), // 65504.0
+ Binary.fromConstantByteArray(new byte[] {(byte) 0x00, (byte) 0x7c}), // Infinity
+ Binary.fromConstantByteArray(new byte[] {0x01, 0x7c}), // +NaN (smallest)
+ Binary.fromConstantByteArray(new byte[] {(byte) 0xff, 0x7f}) // +NaN (largest)
+ };
+
+ for (int i = 0; i < valuesInAscendingOrder.length; ++i) {
+ for (int j = 0; j < valuesInAscendingOrder.length; ++j) {
+ Binary vi = valuesInAscendingOrder[i];
+ Binary vj = valuesInAscendingOrder[j];
+ int exp = i - j;
+ assertSignumEquals(vi, vj, exp, BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR.compare(vi, vj));
+ }
+ }
+ }
+
private <T> void testObjectComparator(PrimitiveComparator<T> comparator, T... valuesInAscendingOrder) {
for (int i = 0; i < valuesInAscendingOrder.length; ++i) {
for (int j = 0; j < valuesInAscendingOrder.length; ++j) {
diff --git a/parquet-common/src/main/java/org/apache/parquet/bytes/LittleEndianDataOutputStream.java b/parquet-common/src/main/java/org/apache/parquet/bytes/LittleEndianDataOutputStream.java
index ef6c71bc8..fbc057969 100644
--- a/parquet-common/src/main/java/org/apache/parquet/bytes/LittleEndianDataOutputStream.java
+++ b/parquet-common/src/main/java/org/apache/parquet/bytes/LittleEndianDataOutputStream.java
@@ -187,15 +187,15 @@ public class LittleEndianDataOutputStream extends OutputStream {
* @param v a <code>float</code> value to be written.
* @throws IOException if an I/O error occurs.
* @see java.io.FilterOutputStream#out
- * @see java.lang.Float#floatToIntBits(float)
+ * @see java.lang.Float#floatToRawIntBits(float)
*/
public final void writeFloat(float v) throws IOException {
- writeInt(Float.floatToIntBits(v));
+ writeInt(Float.floatToRawIntBits(v));
}
/**
* Converts the double argument to a <code>long</code> using the
- * <code>doubleToLongBits</code> method in class <code>Double</code>,
+ * <code>doubleToRawLongBits</code> method in class <code>Double</code>,
* and then writes that <code>long</code> value to the underlying
* output stream as an 8-byte quantity, low byte first. If no
* exception is thrown, the counter <code>written</code> is
@@ -204,10 +204,10 @@ public class LittleEndianDataOutputStream extends OutputStream {
* @param v a <code>double</code> value to be written.
* @throws IOException if an I/O error occurs.
* @see java.io.FilterOutputStream#out
- * @see java.lang.Double#doubleToLongBits(double)
+ * @see java.lang.Double#doubleToRawLongBits(double)
*/
public final void writeDouble(double v) throws IOException {
- writeLong(Double.doubleToLongBits(v));
+ writeLong(Double.doubleToRawLongBits(v));
}
public void close() {
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/bloomfilterlevel/BloomFilterImpl.java b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/bloomfilterlevel/BloomFilterImpl.java
index 39babc0ac..b6404013d 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/bloomfilterlevel/BloomFilterImpl.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/bloomfilterlevel/BloomFilterImpl.java
@@ -32,6 +32,10 @@ import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
import org.apache.parquet.hadoop.BloomFilterReader;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.Float16;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,6 +84,9 @@ public class BloomFilterImpl implements FilterPredicate.Visitor<Boolean> {
// must be non-null because of the above check.
return BLOCK_CANNOT_MATCH;
}
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
try {
BloomFilter bloomFilter = bloomFilterReader.readBloomFilter(meta);
@@ -141,6 +148,9 @@ public class BloomFilterImpl implements FilterPredicate.Visitor<Boolean> {
// must be non-null because of the above check.
return BLOCK_CANNOT_MATCH;
}
+ if (containsNaNLiteral(meta, values)) {
+ return BLOCK_MIGHT_MATCH;
+ }
BloomFilter bloomFilter = bloomFilterReader.readBloomFilter(meta);
if (bloomFilter != null) {
@@ -181,6 +191,37 @@ public class BloomFilterImpl implements FilterPredicate.Visitor<Boolean> {
return BLOCK_MIGHT_MATCH;
}
+ private static <T> boolean containsNaNLiteral(ColumnChunkMetaData meta, Set<T> values) {
+ for (T value : values) {
+ if (isNaNLiteral(meta, value)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isNaNLiteral(ColumnChunkMetaData meta, Object value) {
+ PrimitiveTypeName type = meta.getPrimitiveType().getPrimitiveTypeName();
+ switch (type) {
+ case FLOAT:
+ return value instanceof Float && Float.isNaN((Float) value);
+ case DOUBLE:
+ return value instanceof Double && Double.isNaN((Double) value);
+ case FIXED_LEN_BYTE_ARRAY:
+ return isFloat16(meta)
+ && value instanceof Binary
+ && ((Binary) value).length() == 2
+ && Float16.isNaN(((Binary) value).get2BytesLittleEndian());
+ default:
+ return false;
+ }
+ }
+
+ private static boolean isFloat16(ColumnChunkMetaData meta) {
+ return meta.getPrimitiveType().getLogicalTypeAnnotation()
+ instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
+ }
+
@Override
public <T extends Comparable<T>, U extends UserDefinedPredicate<T>> Boolean visit(Operators.UserDefined<T, U> udp) {
return visit(udp, false);
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java
index c6c9f696f..409caeb80 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilter.java
@@ -52,6 +52,9 @@ import org.apache.parquet.filter2.predicate.Operators.UserDefined;
import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.Float16;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,8 +90,51 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
return columns.get(columnPath);
}
+ private static <T> boolean containsNaNLiteral(ColumnChunkMetaData meta, Set<T> values) {
+ for (T value : values) {
+ if (isNaNLiteral(meta, value)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean isNaNLiteral(ColumnChunkMetaData meta, Object value) {
+ PrimitiveTypeName type = meta.getPrimitiveType().getPrimitiveTypeName();
+ switch (type) {
+ case FLOAT:
+ return value instanceof Float && Float.isNaN((Float) value);
+ case DOUBLE:
+ return value instanceof Double && Double.isNaN((Double) value);
+ case FIXED_LEN_BYTE_ARRAY:
+ return isFloat16(meta)
+ && value instanceof Binary
+ && ((Binary) value).length() == 2
+ && Float16.isNaN(((Binary) value).get2BytesLittleEndian());
+ default:
+ return false;
+ }
+ }
+
+ private static boolean isFloat16(ColumnChunkMetaData meta) {
+ return meta.getPrimitiveType().getLogicalTypeAnnotation()
+ instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
+ }
+
+ // Carries decoded dictionary values plus whether raw entries included any NaN.
+ private static class ExpandedDictionary<T extends Comparable<T>> {
+ private final Set<T> values;
+ private final boolean containsNaN;
+
+ private ExpandedDictionary(Set<T> values, boolean containsNaN) {
+ this.values = values;
+ this.containsNaN = containsNaN;
+ }
+ }
+
@SuppressWarnings("unchecked")
- private <T extends Comparable<T>> Set<T> expandDictionary(ColumnChunkMetaData meta) throws IOException {
+ private <T extends Comparable<T>> ExpandedDictionary<T> expandDictionary(ColumnChunkMetaData meta)
+ throws IOException {
ColumnDescriptor col = new ColumnDescriptor(meta.getPath().toArray(), meta.getPrimitiveType(), -1, -1);
DictionaryPage page = dictionaries.readDictionaryPage(col);
@@ -124,11 +170,16 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
Set<T> dictSet = new HashSet<>();
+ boolean containsNaN = false;
for (int i = 0; i <= dict.getMaxId(); i++) {
- dictSet.add((T) dictValueProvider.apply(i));
+ Object value = dictValueProvider.apply(i);
+ dictSet.add((T) value);
+ if (!containsNaN) {
+ containsNaN = isNaNLiteral(meta, value);
+ }
}
- return dictSet;
+ return new ExpandedDictionary<>(dictSet, containsNaN);
}
@Override
@@ -149,6 +200,9 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
// must be non-null because of the above check.
return BLOCK_CANNOT_MATCH;
}
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
// if the chunk has non-dictionary pages, don't bother decoding the
// dictionary because the row group can't be eliminated.
@@ -157,8 +211,8 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
try {
- Set<T> dictSet = expandDictionary(meta);
- if (dictSet != null && !dictSet.contains(value)) {
+ ExpandedDictionary<T> dictionary = expandDictionary(meta);
+ if (dictionary != null && !dictionary.values.contains(value)) {
return BLOCK_CANNOT_MATCH;
}
} catch (IOException e) {
@@ -192,6 +246,9 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
// the non-null test value, so the predicate is true for all rows
return BLOCK_MIGHT_MATCH;
}
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
// if the chunk has non-dictionary pages, don't bother decoding the
// dictionary because the row group can't be eliminated.
@@ -200,11 +257,14 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
try {
- Set<T> dictSet = expandDictionary(meta);
+ ExpandedDictionary<T> dictionary = expandDictionary(meta);
boolean mayContainNull = (meta.getStatistics() == null
|| !meta.getStatistics().isNumNullsSet()
|| meta.getStatistics().getNumNulls() > 0);
- if (dictSet != null && dictSet.size() == 1 && dictSet.contains(value) && !mayContainNull) {
+ if (dictionary != null
+ && dictionary.values.size() == 1
+ && dictionary.values.contains(value)
+ && !mayContainNull) {
return BLOCK_CANNOT_MATCH;
}
} catch (IOException e) {
@@ -232,15 +292,18 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
T value = lt.getValue();
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
try {
- Set<T> dictSet = expandDictionary(meta);
- if (dictSet == null) {
+ ExpandedDictionary<T> dictionary = expandDictionary(meta);
+ if (dictionary == null || dictionary.containsNaN) {
return BLOCK_MIGHT_MATCH;
}
Comparator<T> comparator = meta.getPrimitiveType().comparator();
- for (T entry : dictSet) {
+ for (T entry : dictionary.values) {
if (comparator.compare(value, entry) > 0) {
return BLOCK_MIGHT_MATCH;
}
@@ -272,17 +335,20 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
T value = ltEq.getValue();
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
filterColumn.getColumnPath();
try {
- Set<T> dictSet = expandDictionary(meta);
- if (dictSet == null) {
+ ExpandedDictionary<T> dictionary = expandDictionary(meta);
+ if (dictionary == null || dictionary.containsNaN) {
return BLOCK_MIGHT_MATCH;
}
Comparator<T> comparator = meta.getPrimitiveType().comparator();
- for (T entry : dictSet) {
+ for (T entry : dictionary.values) {
if (comparator.compare(value, entry) >= 0) {
return BLOCK_MIGHT_MATCH;
}
@@ -314,15 +380,18 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
T value = gt.getValue();
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
try {
- Set<T> dictSet = expandDictionary(meta);
- if (dictSet == null) {
+ ExpandedDictionary<T> dictionary = expandDictionary(meta);
+ if (dictionary == null || dictionary.containsNaN) {
return BLOCK_MIGHT_MATCH;
}
Comparator<T> comparator = meta.getPrimitiveType().comparator();
- for (T entry : dictSet) {
+ for (T entry : dictionary.values) {
if (comparator.compare(value, entry) < 0) {
return BLOCK_MIGHT_MATCH;
}
@@ -354,17 +423,20 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
T value = gtEq.getValue();
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
filterColumn.getColumnPath();
try {
- Set<T> dictSet = expandDictionary(meta);
- if (dictSet == null) {
+ ExpandedDictionary<T> dictionary = expandDictionary(meta);
+ if (dictionary == null || dictionary.containsNaN) {
return BLOCK_MIGHT_MATCH;
}
Comparator<T> comparator = meta.getPrimitiveType().comparator();
- for (T entry : dictSet) {
+ for (T entry : dictionary.values) {
if (comparator.compare(value, entry) <= 0) {
return BLOCK_MIGHT_MATCH;
}
@@ -396,6 +468,9 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
// must be non-null because of the above check.
return BLOCK_CANNOT_MATCH;
}
+ if (containsNaNLiteral(meta, values)) {
+ return BLOCK_MIGHT_MATCH;
+ }
// if the chunk has non-dictionary pages, don't bother decoding the
// dictionary because the row group can't be eliminated.
@@ -404,9 +479,9 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
try {
- Set<T> dictSet = expandDictionary(meta);
- if (dictSet != null) {
- return drop(dictSet, values);
+ ExpandedDictionary<T> dictionary = expandDictionary(meta);
+ if (dictionary != null) {
+ return drop(dictionary.values, values);
}
} catch (IOException e) {
LOG.warn("Failed to process dictionary for filter evaluation.", e);
@@ -460,6 +535,9 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
// must be non-null because of the above check.
return BLOCK_MIGHT_MATCH;
}
+ if (containsNaNLiteral(meta, values)) {
+ return BLOCK_MIGHT_MATCH;
+ }
boolean mayContainNull = (meta.getStatistics() == null
|| !meta.getStatistics().isNumNullsSet()
@@ -476,11 +554,11 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
try {
- Set<T> dictSet = expandDictionary(meta);
- if (dictSet != null) {
- if (dictSet.size() > values.size()) return BLOCK_MIGHT_MATCH;
+ ExpandedDictionary<T> dictionary = expandDictionary(meta);
+ if (dictionary != null) {
+ if (dictionary.values.size() > values.size()) return BLOCK_MIGHT_MATCH;
// ROWS_CANNOT_MATCH if no values in the dictionary that are not also in the set
- return values.containsAll(dictSet) ? BLOCK_CANNOT_MATCH : BLOCK_MIGHT_MATCH;
+ return values.containsAll(dictionary.values) ? BLOCK_CANNOT_MATCH : BLOCK_MIGHT_MATCH;
}
} catch (IOException e) {
LOG.warn("Failed to process dictionary for filter evaluation.", e);
@@ -534,12 +612,12 @@ public class DictionaryFilter implements FilterPredicate.Visitor<Boolean> {
}
try {
- Set<T> dictSet = expandDictionary(meta);
- if (dictSet == null) {
+ ExpandedDictionary<T> dictionary = expandDictionary(meta);
+ if (dictionary == null || dictionary.containsNaN) {
return BLOCK_MIGHT_MATCH;
}
- for (T entry : dictSet) {
+ for (T entry : dictionary.values) {
boolean keep = udp.keep(entry);
if ((keep && !inverted) || (!keep && inverted)) return BLOCK_MIGHT_MATCH;
}
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/statisticslevel/StatisticsFilter.java b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/statisticslevel/StatisticsFilter.java
index fb7e0bade..2574c8ea3 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/statisticslevel/StatisticsFilter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/statisticslevel/StatisticsFilter.java
@@ -44,6 +44,11 @@ import org.apache.parquet.filter2.predicate.Operators.UserDefined;
import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.Float16;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
/**
* Applies a {@link org.apache.parquet.filter2.predicate.FilterPredicate} to statistics about a group of
@@ -99,6 +104,110 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
return column.getStatistics().getNumNulls() > 0;
}
+ private static boolean isFloatingPointColumn(ColumnChunkMetaData column) {
+ PrimitiveType type = column.getPrimitiveType();
+ PrimitiveTypeName typeName = type.getPrimitiveTypeName();
+ return typeName == PrimitiveTypeName.FLOAT
+ || typeName == PrimitiveTypeName.DOUBLE
+ || (typeName == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
+ && type.getLogicalTypeAnnotation()
+ instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation);
+ }
+
+ // check if all non-null values are NaN (nan_count + null_count == value_count)
+ private boolean isAllNaNs(ColumnChunkMetaData column) {
+ if (!isFloatingPointColumn(column)) {
+ return false;
+ }
+ Statistics<?> stats = column.getStatistics();
+ return stats.isNanCountSet()
+ && stats.isNumNullsSet()
+ && stats.getNanCount() > 0
+ && stats.getNanCount() + stats.getNumNulls() == column.getValueCount();
+ }
+
+ private boolean hasNaNs(ColumnChunkMetaData column) {
+ if (!isFloatingPointColumn(column)) {
+ return false;
+ }
+ Statistics<?> stats = column.getStatistics();
+ return !stats.isNanCountSet() || stats.getNanCount() > 0;
+ }
+
+ private static boolean isNaNLiteral(ColumnChunkMetaData column, Object value) {
+ if (!isFloatingPointColumn(column)) {
+ return false;
+ }
+ PrimitiveTypeName typeName = column.getPrimitiveType().getPrimitiveTypeName();
+ if (typeName == PrimitiveTypeName.FLOAT) {
+ return Float.isNaN((Float) value);
+ }
+ if (typeName == PrimitiveTypeName.DOUBLE) {
+ return Double.isNaN((Double) value);
+ }
+ // Float16: value is Binary, must be exactly 2 bytes
+ if (!(value instanceof Binary)) {
+ return false;
+ }
+ Binary b = (Binary) value;
+ if (b.length() != 2) {
+ return false;
+ }
+ return Float16.isNaN(b.get2BytesLittleEndian());
+ }
+
+ // check if any of min or max value is NaN
+ private static boolean hasNaNMinMax(ColumnChunkMetaData column, Statistics<?> stats) {
+ if (!isFloatingPointColumn(column) || !stats.hasNonNullValue()) {
+ return false;
+ }
+ PrimitiveTypeName typeName = column.getPrimitiveType().getPrimitiveTypeName();
+ if (typeName == PrimitiveTypeName.FLOAT) {
+ return Float.isNaN((Float) stats.genericGetMin()) || Float.isNaN((Float) stats.genericGetMax());
+ }
+ if (typeName == PrimitiveTypeName.DOUBLE) {
+ return Double.isNaN((Double) stats.genericGetMin()) || Double.isNaN((Double) stats.genericGetMax());
+ }
+ // Float16
+ Object minVal = stats.genericGetMin();
+ Object maxVal = stats.genericGetMax();
+ if (minVal instanceof Binary && maxVal instanceof Binary) {
+ Binary bMin = (Binary) minVal;
+ Binary bMax = (Binary) maxVal;
+ return (bMin.length() == 2 && Float16.isNaN(bMin.get2BytesLittleEndian()))
+ || (bMax.length() == 2 && Float16.isNaN(bMax.get2BytesLittleEndian()));
+ }
+ return false;
+ }
+
+ private static <T> boolean hasNaNLiteral(ColumnChunkMetaData column, Set<T> values) {
+ PrimitiveTypeName typeName = column.getPrimitiveType().getPrimitiveTypeName();
+ boolean isFloat = typeName == PrimitiveTypeName.FLOAT;
+ boolean isDouble = typeName == PrimitiveTypeName.DOUBLE;
+ boolean isFloat16 = typeName == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
+ && column.getPrimitiveType().getLogicalTypeAnnotation()
+ instanceof LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
+ if (!isFloat && !isDouble && !isFloat16) {
+ return false;
+ }
+ for (T v : values) {
+ if (v == null) {
+ continue;
+ }
+ if (isFloat) {
+ if (Float.isNaN((Float) v)) return true;
+ } else if (isDouble) {
+ if (Double.isNaN((Double) v)) return true;
+ } else if (v instanceof Binary) {
+ Binary b = (Binary) v;
+ if (b.length() == 2 && Float16.isNaN(b.get2BytesLittleEndian())) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
@Override
@SuppressWarnings("unchecked")
public <T extends Comparable<T>> Boolean visit(Eq<T> eq) {
@@ -139,7 +248,15 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
return BLOCK_CANNOT_MATCH;
}
- if (!stats.hasNonNullValue()) {
+ if (isNaNLiteral(meta, value)) {
+ return (!stats.isNanCountSet() || stats.getNanCount() > 0) ? BLOCK_MIGHT_MATCH : BLOCK_CANNOT_MATCH;
+ }
+
+ if (isAllNaNs(meta)) {
+ return BLOCK_CANNOT_MATCH;
+ }
+
+ if (!stats.hasNonNullValue() || hasNaNMinMax(meta, stats)) {
// stats does not contain min/max values, we cannot drop any chunks
return BLOCK_MIGHT_MATCH;
}
@@ -182,11 +299,6 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
}
}
- if (!stats.hasNonNullValue()) {
- // stats does not contain min/max values, we cannot drop any chunks
- return BLOCK_MIGHT_MATCH;
- }
-
if (stats.isNumNullsSet()) {
if (stats.getNumNulls() == 0) {
if (values.contains(null) && values.size() == 1) return BLOCK_CANNOT_MATCH;
@@ -195,6 +307,20 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
}
}
+ // If any value in the IN set is NaN, be conservative
+ if (hasNaNLiteral(meta, values)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
+ if (isAllNaNs(meta)) {
+ return (values.contains(null) && hasNulls(meta)) ? BLOCK_MIGHT_MATCH : BLOCK_CANNOT_MATCH;
+ }
+
+ if (!stats.hasNonNullValue() || hasNaNMinMax(meta, stats)) {
+ // stats does not contain min/max values, we cannot drop any chunks
+ return BLOCK_MIGHT_MATCH;
+ }
+
MinMax<T> minMax = new MinMax(meta.getPrimitiveType().comparator(), values);
T min = minMax.getMin();
T max = minMax.getMax();
@@ -252,7 +378,22 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
return BLOCK_MIGHT_MATCH;
}
- if (!stats.hasNonNullValue()) {
+ if (isNaNLiteral(meta, value)) {
+ // We are looking for records where v != NaN. Every non-NaN value (and every null) satisfies
+ // this predicate, and the chunk-level statistics cannot prove that all values are NaN in a
+ // way that lets us drop the block safely, so we conservatively keep it.
+ return BLOCK_MIGHT_MATCH;
+ }
+
+ if (isAllNaNs(meta)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
+ if (hasNaNs(meta)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
+ if (!stats.hasNonNullValue() || hasNaNMinMax(meta, stats)) {
// stats does not contain min/max values, we cannot drop any chunks
return BLOCK_MIGHT_MATCH;
}
@@ -286,13 +427,21 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
return BLOCK_CANNOT_MATCH;
}
- if (!stats.hasNonNullValue()) {
+ if (!stats.hasNonNullValue() || hasNaNMinMax(meta, stats)) {
// stats does not contain min/max values, we cannot drop any chunks
return BLOCK_MIGHT_MATCH;
}
T value = lt.getValue();
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
+ if (hasNaNs(meta)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
// drop if value <= min
return stats.compareMinToValue(value) >= 0;
}
@@ -322,13 +471,21 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
return BLOCK_CANNOT_MATCH;
}
- if (!stats.hasNonNullValue()) {
+ if (!stats.hasNonNullValue() || hasNaNMinMax(meta, stats)) {
// stats does not contain min/max values, we cannot drop any chunks
return BLOCK_MIGHT_MATCH;
}
T value = ltEq.getValue();
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
+ if (hasNaNs(meta)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
// drop if value < min
return stats.compareMinToValue(value) > 0;
}
@@ -358,13 +515,21 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
return BLOCK_CANNOT_MATCH;
}
- if (!stats.hasNonNullValue()) {
+ if (!stats.hasNonNullValue() || hasNaNMinMax(meta, stats)) {
// stats does not contain min/max values, we cannot drop any chunks
return BLOCK_MIGHT_MATCH;
}
T value = gt.getValue();
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
+ if (hasNaNs(meta)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
// drop if value >= max
return stats.compareMaxToValue(value) <= 0;
}
@@ -394,13 +559,21 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
return BLOCK_CANNOT_MATCH;
}
- if (!stats.hasNonNullValue()) {
+ if (!stats.hasNonNullValue() || hasNaNMinMax(meta, stats)) {
// stats does not contain min/max values, we cannot drop any chunks
return BLOCK_MIGHT_MATCH;
}
T value = gtEq.getValue();
+ if (isNaNLiteral(meta, value)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
+ if (hasNaNs(meta)) {
+ return BLOCK_MIGHT_MATCH;
+ }
+
// drop if value > max
return stats.compareMaxToValue(value) < 0;
}
@@ -462,7 +635,7 @@ public class StatisticsFilter implements FilterPredicate.Visitor<Boolean> {
}
}
- if (!stats.hasNonNullValue()) {
+ if (!stats.hasNonNullValue() || hasNaNMinMax(columnChunk, stats) || hasNaNs(columnChunk)) {
// stats does not contain min/max values, we cannot drop any chunks
return BLOCK_MIGHT_MATCH;
}
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
index 89e39c3b6..50c2e344e 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.CorruptStatistics;
import org.apache.parquet.ParquetReadOptions;
import org.apache.parquet.Preconditions;
+import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.EncodingStats;
import org.apache.parquet.column.ParquetProperties;
import org.apache.parquet.column.statistics.BinaryStatistics;
@@ -86,6 +87,7 @@ import org.apache.parquet.format.FileMetaData;
import org.apache.parquet.format.GeographyType;
import org.apache.parquet.format.GeometryType;
import org.apache.parquet.format.GeospatialStatistics;
+import org.apache.parquet.format.IEEE754TotalOrder;
import org.apache.parquet.format.IntType;
import org.apache.parquet.format.KeyValue;
import org.apache.parquet.format.LogicalType;
@@ -143,6 +145,7 @@ import org.slf4j.LoggerFactory;
public class ParquetMetadataConverter {
private static final TypeDefinedOrder TYPE_DEFINED_ORDER = new TypeDefinedOrder();
+ private static final IEEE754TotalOrder IEEE_754_TOTAL_ORDER = new IEEE754TotalOrder();
public static final MetadataFilter NO_FILTER = new NoFilter();
public static final MetadataFilter SKIP_ROW_GROUPS = new SkipMetadataFilter();
public static final long MAX_STATS_SIZE = 4096; // limit stats to 4k
@@ -278,11 +281,23 @@ public class ParquetMetadataConverter {
private List<ColumnOrder> getColumnOrders(MessageType schema) {
List<ColumnOrder> columnOrders = new ArrayList<>();
- // Currently, only TypeDefinedOrder is supported, so we create a column order for each columns with
- // TypeDefinedOrder even if some types (e.g. INT96) have undefined column orders.
- for (int i = 0, n = schema.getPaths().size(); i < n; ++i) {
+ for (ColumnDescriptor column : schema.getColumns()) {
ColumnOrder columnOrder = new ColumnOrder();
- columnOrder.setTYPE_ORDER(TYPE_DEFINED_ORDER);
+ switch (column.getPrimitiveType().columnOrder().getColumnOrderName()) {
+ case TYPE_DEFINED_ORDER:
+ columnOrder.setTYPE_ORDER(TYPE_DEFINED_ORDER);
+ break;
+ case IEEE_754_TOTAL_ORDER:
+ columnOrder.setIEEE_754_TOTAL_ORDER(IEEE_754_TOTAL_ORDER);
+ break;
+ case UNDEFINED:
+ // Use TypeDefinedOrder if some types (e.g. INT96) have undefined column orders.
+ columnOrder.setTYPE_ORDER(TYPE_DEFINED_ORDER);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unknown column order: " + column.getPrimitiveType().columnOrder());
+ }
columnOrders.add(columnOrder);
}
return columnOrders;
@@ -802,6 +817,9 @@ public class ParquetMetadataConverter {
Statistics formatStats = new Statistics();
if (!stats.isEmpty()) {
formatStats.setNull_count(stats.getNumNulls());
+ if (stats.isNanCountSet()) {
+ formatStats.setNan_count(stats.getNanCount());
+ }
}
// Don't write stats larger than the max size rather than truncating. The
// rationale is that some engines may use the minimum value in the page as
@@ -893,7 +911,8 @@ public class ParquetMetadataConverter {
}
private static boolean isMinMaxStatsSupported(PrimitiveType type) {
- return type.columnOrder().getColumnOrderName() == ColumnOrderName.TYPE_DEFINED_ORDER;
+ return type.columnOrder().getColumnOrderName() == ColumnOrderName.TYPE_DEFINED_ORDER
+ || type.columnOrder().getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER;
}
/**
@@ -962,6 +981,9 @@ public class ParquetMetadataConverter {
if (formatStats.isSetNull_count()) {
statsBuilder.withNumNulls(formatStats.null_count);
}
+ if (formatStats.isSetNan_count()) {
+ statsBuilder.withNanCount(formatStats.getNan_count());
+ }
}
return statsBuilder.build();
}
@@ -1998,7 +2020,8 @@ public class ParquetMetadataConverter {
if (root.isSetField_id()) {
builder.id(root.field_id);
}
- buildChildren(builder, iterator, root.getNum_children(), columnOrders, 0);
+ Iterator<ColumnOrder> columnOrderIterator = columnOrders == null ? null : columnOrders.iterator();
+ buildChildren(builder, iterator, root.getNum_children(), columnOrderIterator);
return builder.named(root.name);
}
@@ -2006,8 +2029,7 @@ public class ParquetMetadataConverter {
Types.GroupBuilder builder,
Iterator<SchemaElement> schema,
int childrenCount,
- List<ColumnOrder> columnOrders,
- int columnCount) {
+ Iterator<ColumnOrder> columnOrders) {
for (int i = 0; i < childrenCount; i++) {
SchemaElement schemaElement = schema.next();
@@ -2026,8 +2048,7 @@ public class ParquetMetadataConverter {
primitiveBuilder.scale(schemaElement.scale);
}
if (columnOrders != null) {
- org.apache.parquet.schema.ColumnOrder columnOrder =
- fromParquetColumnOrder(columnOrders.get(columnCount));
+ org.apache.parquet.schema.ColumnOrder columnOrder = fromParquetColumnOrder(columnOrders.next());
// As per parquet format 2.4.0 no UNDEFINED order is supported. So, set undefined column order for
// the types
// where ordering is not supported.
@@ -2041,12 +2062,7 @@ public class ParquetMetadataConverter {
childBuilder = primitiveBuilder;
} else {
childBuilder = builder.group(fromParquetRepetition(schemaElement.repetition_type));
- buildChildren(
- (Types.GroupBuilder) childBuilder,
- schema,
- schemaElement.num_children,
- columnOrders,
- columnCount);
+ buildChildren((Types.GroupBuilder) childBuilder, schema, schemaElement.num_children, columnOrders);
}
if (schemaElement.isSetLogicalType()) {
@@ -2074,7 +2090,6 @@ public class ParquetMetadataConverter {
}
childBuilder.named(schemaElement.name);
- ++columnCount;
}
}
@@ -2092,6 +2107,9 @@ public class ParquetMetadataConverter {
if (columnOrder.isSetTYPE_ORDER()) {
return org.apache.parquet.schema.ColumnOrder.typeDefined();
}
+ if (columnOrder.isSetIEEE_754_TOTAL_ORDER()) {
+ return org.apache.parquet.schema.ColumnOrder.ieee754TotalOrder();
+ }
// The column order is not yet supported by this API
return org.apache.parquet.schema.ColumnOrder.undefined();
}
@@ -2551,6 +2569,10 @@ public class ParquetMetadataConverter {
columnIndex.getMaxValues(),
toParquetBoundaryOrder(columnIndex.getBoundaryOrder()));
parquetColumnIndex.setNull_counts(columnIndex.getNullCounts());
+ List<Long> nanCounts = columnIndex.getNanCounts();
+ if (nanCounts != null && !nanCounts.isEmpty()) {
+ parquetColumnIndex.setNan_counts(nanCounts);
+ }
List<Long> repLevelHistogram = columnIndex.getRepetitionLevelHistogram();
if (repLevelHistogram != null && !repLevelHistogram.isEmpty()) {
parquetColumnIndex.setRepetition_level_histograms(repLevelHistogram);
@@ -2572,6 +2594,7 @@ public class ParquetMetadataConverter {
fromParquetBoundaryOrder(parquetColumnIndex.getBoundary_order()),
parquetColumnIndex.getNull_pages(),
parquetColumnIndex.getNull_counts(),
+ parquetColumnIndex.getNan_counts(),
parquetColumnIndex.getMin_values(),
parquetColumnIndex.getMax_values(),
parquetColumnIndex.getRepetition_level_histograms(),
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java
index cd82cf4a8..88e41626d 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java
@@ -810,6 +810,10 @@ public class ParquetRewriter implements Closeable {
org.apache.parquet.column.statistics.Statistics.Builder statsBuilder =
org.apache.parquet.column.statistics.Statistics.getBuilderForReading(type);
statsBuilder.withNumNulls(columnIndex.getNullCounts().get(pageIndex));
+ List<Long> nanCounts = columnIndex.getNanCounts();
+ if (nanCounts != null && pageIndex < nanCounts.size()) {
+ statsBuilder.withNanCount(nanCounts.get(pageIndex));
+ }
if (!columnIndex.getNullPages().get(pageIndex)) {
statsBuilder.withMin(
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/bloomfilterlevel/TestBloomFilterImpl.java b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/bloomfilterlevel/TestBloomFilterImpl.java
new file mode 100644
index 000000000..f2a0cfe1f
--- /dev/null
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/bloomfilterlevel/TestBloomFilterImpl.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.filter2.bloomfilterlevel;
+
+import static org.apache.parquet.filter2.predicate.FilterApi.binaryColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.doubleColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.eq;
+import static org.apache.parquet.filter2.predicate.FilterApi.floatColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.in;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.parquet.column.Encoding;
+import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.column.values.bloomfilter.BlockSplitBloomFilter;
+import org.apache.parquet.column.values.bloomfilter.BloomFilter;
+import org.apache.parquet.filter2.predicate.Operators.BinaryColumn;
+import org.apache.parquet.filter2.predicate.Operators.DoubleColumn;
+import org.apache.parquet.filter2.predicate.Operators.FloatColumn;
+import org.apache.parquet.hadoop.BloomFilterReader;
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+import org.junit.Test;
+
+public class TestBloomFilterImpl {
+ private static final double DOUBLE_NAN = Double.longBitsToDouble(0x7ff8000000000001L);
+ private static final float FLOAT_NAN = Float.intBitsToFloat(0x7fc00001);
+ private static final Binary FLOAT16_NAN = Binary.fromConstantByteArray(new byte[] {0x01, 0x7e});
+
+ @Test
+ public void testDoubleNaNLiteralDoesNotDrop() {
+ DoubleColumn column = doubleColumn("double_column");
+ ColumnChunkMetaData meta = columnMeta("double_column", PrimitiveTypeName.DOUBLE);
+ BloomFilterReader reader = bloomFilterReader(meta, new BlockSplitBloomFilter(0));
+
+ assertTrue(BloomFilterImpl.canDrop(eq(column, 1.0D), List.of(meta), reader));
+ assertFalse(BloomFilterImpl.canDrop(eq(column, DOUBLE_NAN), List.of(meta), reader));
+
+ Set<Double> values = new HashSet<>();
+ values.add(DOUBLE_NAN);
+ assertFalse(BloomFilterImpl.canDrop(in(column, values), List.of(meta), reader));
+ }
+
+ @Test
+ public void testFloatNaNLiteralDoesNotDrop() {
+ FloatColumn column = floatColumn("float_column");
+ ColumnChunkMetaData meta = columnMeta("float_column", PrimitiveTypeName.FLOAT);
+ BloomFilterReader reader = bloomFilterReader(meta, new BlockSplitBloomFilter(0));
+
+ assertTrue(BloomFilterImpl.canDrop(eq(column, 1.0F), List.of(meta), reader));
+ assertFalse(BloomFilterImpl.canDrop(eq(column, FLOAT_NAN), List.of(meta), reader));
+
+ Set<Float> values = new HashSet<>();
+ values.add(FLOAT_NAN);
+ assertFalse(BloomFilterImpl.canDrop(in(column, values), List.of(meta), reader));
+ }
+
+ @Test
+ public void testFloat16NaNLiteralDoesNotDrop() {
+ BinaryColumn column = binaryColumn("float16_column");
+ PrimitiveType type = Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(LogicalTypeAnnotation.float16Type())
+ .named("float16_column");
+ ColumnChunkMetaData meta = columnMeta("float16_column", type);
+ BloomFilterReader reader = bloomFilterReader(meta, new BlockSplitBloomFilter(0));
+
+ assertFalse(BloomFilterImpl.canDrop(eq(column, FLOAT16_NAN), List.of(meta), reader));
+
+ Set<Binary> values = new HashSet<>();
+ values.add(FLOAT16_NAN);
+ assertFalse(BloomFilterImpl.canDrop(in(column, values), List.of(meta), reader));
+ }
+
+ private static ColumnChunkMetaData columnMeta(String name, PrimitiveTypeName type) {
+ return columnMeta(name, Types.required(type).named(name));
+ }
+
+ private static ColumnChunkMetaData columnMeta(String name, PrimitiveType type) {
+ Statistics<?> stats = Statistics.getBuilderForReading(type).build();
+ return ColumnChunkMetaData.get(
+ ColumnPath.get(name),
+ type,
+ CompressionCodecName.UNCOMPRESSED,
+ null,
+ new HashSet<>(List.of(Encoding.PLAIN)),
+ stats,
+ 0L,
+ 0L,
+ 1L,
+ 0L,
+ 0L);
+ }
+
+ private static BloomFilterReader bloomFilterReader(ColumnChunkMetaData meta, BloomFilter bloomFilter) {
+ BlockMetaData block = new BlockMetaData();
+ block.addColumn(meta);
+ return new BloomFilterReader(null, block) {
+ @Override
+ public BloomFilter readBloomFilter(ColumnChunkMetaData requestedMeta) {
+ return requestedMeta == meta ? bloomFilter : null;
+ }
+ };
+ }
+}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilterTest.java b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilterTest.java
index 981a3820b..75df3a8a6 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilterTest.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/dictionarylevel/DictionaryFilterTest.java
@@ -53,6 +53,8 @@ import com.google.common.primitives.Ints;
import java.io.IOException;
import java.io.Serializable;
import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -61,9 +63,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.FixedBinaryTestUtils;
+import org.apache.parquet.bytes.BytesInput;
+import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.EncodingStats;
import org.apache.parquet.column.ParquetProperties.WriterVersion;
+import org.apache.parquet.column.page.DictionaryPage;
import org.apache.parquet.column.page.DictionaryPageReadStore;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
@@ -82,9 +87,16 @@ import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
import org.apache.parquet.hadoop.example.GroupWriteSupport;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -101,6 +113,14 @@ public class DictionaryFilterTest {
private static final Configuration conf = new Configuration();
private static final Path FILE_V1 = new Path("target/test/TestDictionaryFilter/testParquetFileV1.parquet");
private static final Path FILE_V2 = new Path("target/test/TestDictionaryFilter/testParquetFileV2.parquet");
+ private static final double DOUBLE_NAN_A = Double.longBitsToDouble(0x7ff8000000000001L);
+ private static final double DOUBLE_NAN_B = Double.longBitsToDouble(0x7ff8000000000002L);
+ private static final double NEGATIVE_DOUBLE_NAN = Double.longBitsToDouble(0xfff8000000000001L);
+ private static final float FLOAT_NAN_A = Float.intBitsToFloat(0x7fc00001);
+ private static final float FLOAT_NAN_B = Float.intBitsToFloat(0x7fc00002);
+ private static final float NEGATIVE_FLOAT_NAN = Float.intBitsToFloat(0xffc00001);
+ private static final Binary FLOAT16_NAN_A = Binary.fromConstantByteArray(new byte[] {0x01, 0x7e});
+ private static final Binary FLOAT16_NAN_B = Binary.fromConstantByteArray(new byte[] {0x02, 0x7e});
private static final MessageType schema = parseMessageType("message test { "
+ "required binary binary_field; "
+ "required binary single_value_field; "
@@ -116,7 +136,6 @@ public class DictionaryFilterTest {
+ "required int96 int96_field; "
+ "repeated binary repeated_binary_field;"
+ "} ");
-
private static final String ALPHABET = "abcdefghijklmnopqrstuvwxyz";
private static final int[] intValues = new int[] {
-100, 302, 3333333, 7654321, 1234567, -2000, -77775, 0, 75, 22223,
@@ -509,6 +528,196 @@ public class DictionaryFilterTest {
"Should not drop: contains matching values", canDrop(gtEq(d, Double.MIN_VALUE), ccmd, dictionaries));
}
+ @Test
+ public void testNaNDictionaryFilterIsConservative() throws Exception {
+ List<ColumnChunkMetaData> nanColumns = nanColumns();
+ DictionaryPageReadStore nanDictionaries = nanDictionaries();
+ DoubleColumn doubleColumn = doubleColumn("double_nan_field");
+ FloatColumn floatColumn = floatColumn("float_nan_field");
+ BinaryColumn float16Column = binaryColumn("float16_nan_field");
+
+ assertFalse(canDrop(eq(doubleColumn, DOUBLE_NAN_A), nanColumns, nanDictionaries));
+ assertFalse(canDrop(eq(doubleColumn, DOUBLE_NAN_B), nanColumns, nanDictionaries));
+ assertFalse(canDrop(notEq(doubleColumn, DOUBLE_NAN_A), nanColumns, nanDictionaries));
+ assertFalse(canDrop(notEq(doubleColumn, DOUBLE_NAN_B), nanColumns, nanDictionaries));
+ assertFalse(canDrop(lt(doubleColumn, DOUBLE_NAN_B), nanColumns, nanDictionaries));
+ assertFalse(canDrop(gt(doubleColumn, DOUBLE_NAN_B), nanColumns, nanDictionaries));
+
+ assertFalse(canDrop(eq(floatColumn, FLOAT_NAN_A), nanColumns, nanDictionaries));
+ assertFalse(canDrop(eq(floatColumn, FLOAT_NAN_B), nanColumns, nanDictionaries));
+ assertFalse(canDrop(notEq(floatColumn, FLOAT_NAN_A), nanColumns, nanDictionaries));
+ assertFalse(canDrop(notEq(floatColumn, FLOAT_NAN_B), nanColumns, nanDictionaries));
+ assertFalse(canDrop(lt(floatColumn, FLOAT_NAN_B), nanColumns, nanDictionaries));
+ assertFalse(canDrop(gt(floatColumn, FLOAT_NAN_B), nanColumns, nanDictionaries));
+
+ Set<Double> doubleSet = new HashSet<>();
+ doubleSet.add(DOUBLE_NAN_B);
+ assertFalse(canDrop(in(doubleColumn, doubleSet), nanColumns, nanDictionaries));
+ assertFalse(canDrop(notIn(doubleColumn, doubleSet), nanColumns, nanDictionaries));
+
+ Set<Float> floatSet = new HashSet<>();
+ floatSet.add(FLOAT_NAN_B);
+ assertFalse(canDrop(in(floatColumn, floatSet), nanColumns, nanDictionaries));
+ assertFalse(canDrop(notIn(floatColumn, floatSet), nanColumns, nanDictionaries));
+
+ Set<Binary> float16Set = new HashSet<>();
+ float16Set.add(FLOAT16_NAN_B);
+ assertFalse(canDrop(in(float16Column, float16Set), nanColumns, nanDictionaries));
+ assertFalse(canDrop(notIn(float16Column, float16Set), nanColumns, nanDictionaries));
+
+ assertFalse(canDrop(eq(float16Column, FLOAT16_NAN_A), nanColumns, nanDictionaries));
+ assertFalse(canDrop(eq(float16Column, FLOAT16_NAN_B), nanColumns, nanDictionaries));
+ assertFalse(canDrop(notEq(float16Column, FLOAT16_NAN_A), nanColumns, nanDictionaries));
+ assertFalse(canDrop(notEq(float16Column, FLOAT16_NAN_B), nanColumns, nanDictionaries));
+ assertFalse(canDrop(lt(float16Column, FLOAT16_NAN_B), nanColumns, nanDictionaries));
+ assertFalse(canDrop(gt(float16Column, FLOAT16_NAN_B), nanColumns, nanDictionaries));
+ }
+
+ @Test
+ public void testNaNDictionaryValuesMakeRangeFiltersConservative() throws Exception {
+ DictionaryPageReadStore dictionariesWithNaNs = dictionariesWithNaNs();
+ assertNaNDictionaryRangeFiltersAreConservative(nanColumns(), dictionariesWithNaNs);
+ assertNaNDictionaryRangeFiltersAreConservative(ieeeNaNColumns(), dictionariesWithNaNs);
+ }
+
+ private static void assertNaNDictionaryRangeFiltersAreConservative(
+ List<ColumnChunkMetaData> nanColumns, DictionaryPageReadStore dictionariesWithNaNs) throws IOException {
+ DoubleColumn doubleColumn = doubleColumn("double_nan_field");
+ FloatColumn floatColumn = floatColumn("float_nan_field");
+ BinaryColumn float16Column = binaryColumn("float16_nan_field");
+
+ assertFalse(canDrop(gt(doubleColumn, 0.0D), nanColumns, dictionariesWithNaNs));
+ assertFalse(canDrop(gtEq(doubleColumn, 0.0D), nanColumns, dictionariesWithNaNs));
+ assertFalse(canDrop(lt(doubleColumn, 0.0D), nanColumns, dictionariesWithNaNs));
+ assertFalse(canDrop(ltEq(doubleColumn, 0.0D), nanColumns, dictionariesWithNaNs));
+
+ assertFalse(canDrop(gt(floatColumn, 0.0F), nanColumns, dictionariesWithNaNs));
+ assertFalse(canDrop(gtEq(floatColumn, 0.0F), nanColumns, dictionariesWithNaNs));
+ assertFalse(canDrop(lt(floatColumn, 0.0F), nanColumns, dictionariesWithNaNs));
+ assertFalse(canDrop(ltEq(floatColumn, 0.0F), nanColumns, dictionariesWithNaNs));
+
+ Binary zero = Binary.fromConstantByteArray(new byte[] {0x00, 0x00});
+ assertFalse(canDrop(gt(float16Column, zero), nanColumns, dictionariesWithNaNs));
+ assertFalse(canDrop(gtEq(float16Column, zero), nanColumns, dictionariesWithNaNs));
+ assertFalse(canDrop(lt(float16Column, zero), nanColumns, dictionariesWithNaNs));
+ assertFalse(canDrop(ltEq(float16Column, zero), nanColumns, dictionariesWithNaNs));
+ }
+
+ private static List<ColumnChunkMetaData> nanColumns() {
+ return List.of(
+ nanColumn(
+ "double_nan_field",
+ Types.required(PrimitiveTypeName.DOUBLE).named("double_nan_field")),
+ nanColumn(
+ "float_nan_field",
+ Types.required(PrimitiveTypeName.FLOAT).named("float_nan_field")),
+ nanColumn(
+ "float16_nan_field",
+ Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(LogicalTypeAnnotation.float16Type())
+ .named("float16_nan_field")));
+ }
+
+ private static List<ColumnChunkMetaData> ieeeNaNColumns() {
+ return List.of(
+ nanColumn(
+ "double_nan_field",
+ Types.required(PrimitiveTypeName.DOUBLE)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("double_nan_field")),
+ nanColumn(
+ "float_nan_field",
+ Types.required(PrimitiveTypeName.FLOAT)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("float_nan_field")),
+ nanColumn(
+ "float16_nan_field",
+ Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(LogicalTypeAnnotation.float16Type())
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("float16_nan_field")));
+ }
+
+ private static ColumnChunkMetaData nanColumn(String name, PrimitiveType type) {
+ EncodingStats encodingStats = new EncodingStats.Builder()
+ .addDictEncoding(Encoding.PLAIN)
+ .addDataEncoding(Encoding.RLE_DICTIONARY)
+ .build();
+ org.apache.parquet.column.statistics.Statistics<?> stats =
+ org.apache.parquet.column.statistics.Statistics.getBuilderForReading(type)
+ .build();
+ stats.setNumNulls(0);
+ return ColumnChunkMetaData.get(
+ ColumnPath.get(name),
+ type,
+ CompressionCodecName.UNCOMPRESSED,
+ encodingStats,
+ new HashSet<>(List.of(Encoding.PLAIN, Encoding.RLE_DICTIONARY)),
+ stats,
+ 0L,
+ 0L,
+ 1L,
+ 0L,
+ 0L);
+ }
+
+ private static DictionaryPageReadStore nanDictionaries() {
+ return new DictionaryPageReadStore() {
+ @Override
+ public DictionaryPage readDictionaryPage(ColumnDescriptor descriptor) {
+ throw new AssertionError("NaN literals should not read dictionary pages");
+ }
+ };
+ }
+
+ private static DictionaryPageReadStore dictionariesWithNaNs() {
+ return new DictionaryPageReadStore() {
+ @Override
+ public DictionaryPage readDictionaryPage(ColumnDescriptor descriptor) {
+ PrimitiveTypeName type = descriptor.getPrimitiveType().getPrimitiveTypeName();
+ switch (type) {
+ case DOUBLE:
+ return new DictionaryPage(
+ BytesInput.from(littleEndianLongs(
+ Double.doubleToRawLongBits(NEGATIVE_DOUBLE_NAN),
+ Double.doubleToRawLongBits(DOUBLE_NAN_A))),
+ 2,
+ Encoding.PLAIN);
+ case FLOAT:
+ return new DictionaryPage(
+ BytesInput.from(littleEndianInts(
+ Float.floatToRawIntBits(NEGATIVE_FLOAT_NAN),
+ Float.floatToRawIntBits(FLOAT_NAN_A))),
+ 2,
+ Encoding.PLAIN);
+ case FIXED_LEN_BYTE_ARRAY:
+ return new DictionaryPage(
+ BytesInput.from(new byte[] {(byte) 0x01, (byte) 0xfe, 0x01, 0x7e}), 2, Encoding.PLAIN);
+ default:
+ throw new AssertionError("Unexpected dictionary type: " + type);
+ }
+ }
+ };
+ }
+
+ private static byte[] littleEndianLongs(long... values) {
+ ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES * values.length).order(ByteOrder.LITTLE_ENDIAN);
+ for (long value : values) {
+ buffer.putLong(value);
+ }
+ return buffer.array();
+ }
+
+ private static byte[] littleEndianInts(int... values) {
+ ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES * values.length).order(ByteOrder.LITTLE_ENDIAN);
+ for (int value : values) {
+ buffer.putInt(value);
+ }
+ return buffer.array();
+ }
+
@Test
public void testInBinary() throws Exception {
BinaryColumn b = binaryColumn("binary_field");
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/statisticslevel/TestStatisticsFilter.java b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/statisticslevel/TestStatisticsFilter.java
index 6e1f267e8..e9756fa82 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/statisticslevel/TestStatisticsFilter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/statisticslevel/TestStatisticsFilter.java
@@ -23,6 +23,7 @@ import static org.apache.parquet.filter2.predicate.FilterApi.binaryColumn;
import static org.apache.parquet.filter2.predicate.FilterApi.contains;
import static org.apache.parquet.filter2.predicate.FilterApi.doubleColumn;
import static org.apache.parquet.filter2.predicate.FilterApi.eq;
+import static org.apache.parquet.filter2.predicate.FilterApi.floatColumn;
import static org.apache.parquet.filter2.predicate.FilterApi.gt;
import static org.apache.parquet.filter2.predicate.FilterApi.gtEq;
import static org.apache.parquet.filter2.predicate.FilterApi.in;
@@ -44,14 +45,17 @@ import static org.junit.Assert.fail;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.parquet.bytes.BytesUtils;
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.statistics.DoubleStatistics;
+import org.apache.parquet.column.statistics.FloatStatistics;
import org.apache.parquet.column.statistics.IntStatistics;
import org.apache.parquet.filter2.predicate.FilterPredicate;
import org.apache.parquet.filter2.predicate.LogicalInverseRewriter;
import org.apache.parquet.filter2.predicate.Operators;
import org.apache.parquet.filter2.predicate.Operators.BinaryColumn;
import org.apache.parquet.filter2.predicate.Operators.DoubleColumn;
+import org.apache.parquet.filter2.predicate.Operators.FloatColumn;
import org.apache.parquet.filter2.predicate.Operators.IntColumn;
import org.apache.parquet.filter2.predicate.Statistics;
import org.apache.parquet.filter2.predicate.UserDefinedPredicate;
@@ -59,6 +63,9 @@ import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
import org.apache.parquet.hadoop.metadata.ColumnPath;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Types;
import org.junit.Test;
@@ -589,4 +596,404 @@ public class TestStatisticsFilter {
e.getMessage());
}
}
+
+ private static final FloatColumn floatCol = floatColumn("float.column");
+
+ private static ColumnChunkMetaData getFloatColumnMeta(
+ org.apache.parquet.column.statistics.Statistics<?> stats, long valueCount) {
+ return ColumnChunkMetaData.get(
+ ColumnPath.get("float", "column"),
+ PrimitiveTypeName.FLOAT,
+ CompressionCodecName.GZIP,
+ new HashSet<>(List.of(Encoding.PLAIN)),
+ stats,
+ 0L,
+ 0L,
+ valueCount,
+ 0L,
+ 0L);
+ }
+
+ private static ColumnChunkMetaData getDoubleColumnMetaWithType(
+ PrimitiveType type, org.apache.parquet.column.statistics.Statistics<?> stats, long valueCount) {
+ return ColumnChunkMetaData.get(
+ ColumnPath.get("double", "column"),
+ type,
+ CompressionCodecName.GZIP,
+ null,
+ new HashSet<>(List.of(Encoding.PLAIN)),
+ stats,
+ 0L,
+ 0L,
+ valueCount,
+ 0L,
+ 0L);
+ }
+
+ // ========================= Double NaN Tests =========================
+
+ @Test
+ public void testNaNDoubleAllNaN() {
+ // All non-null values are NaN, TYPE_DEFINED_ORDER (no min/max set)
+ DoubleStatistics allNanStats = new DoubleStatistics();
+ allNanStats.setNumNulls(0);
+ allNanStats.incrementNanCount(177);
+
+ List<ColumnChunkMetaData> metas =
+ List.of(getIntColumnMeta(intStats, 177L), getDoubleColumnMeta(allNanStats, 177L));
+
+ assertTrue(canDrop(eq(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(notEq(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(lt(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(ltEq(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(gt(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(gtEq(doubleColumn, 5.0), metas));
+ assertTrue(canDrop(in(doubleColumn, new HashSet<>(List.of(5.0))), metas));
+
+ assertFalse(canDrop(eq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(notEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(lt(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(ltEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(gt(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(gtEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(in(doubleColumn, new HashSet<>(List.of(Double.NaN))), metas));
+ }
+
+ @Test
+ public void testNaNDoubleMixed() {
+ // Mixed: nan_count=50, null_count=0, min=10, max=100, valueCount=177
+ DoubleStatistics mixedStats = new DoubleStatistics();
+ mixedStats.setMinMax(10, 100);
+ mixedStats.setNumNulls(0);
+ mixedStats.incrementNanCount(50);
+
+ List<ColumnChunkMetaData> metas =
+ List.of(getIntColumnMeta(intStats, 177L), getDoubleColumnMeta(mixedStats, 177L));
+
+ // Non-NaN literal within range: cannot drop
+ assertFalse(canDrop(eq(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(notEq(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(lt(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(ltEq(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(gt(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(gtEq(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(in(doubleColumn, new HashSet<>(List.of(50.0))), metas));
+ assertFalse(canDrop(lt(doubleColumn, 0.0), metas));
+ assertFalse(canDrop(gt(doubleColumn, 200.0), metas));
+
+ DoubleStatistics mixedEqualStats = new DoubleStatistics();
+ mixedEqualStats.setMinMax(5.0, 5.0);
+ mixedEqualStats.setNumNulls(0);
+ mixedEqualStats.incrementNanCount(1);
+ List<ColumnChunkMetaData> mixedEqualMetas =
+ List.of(getIntColumnMeta(intStats, 177L), getDoubleColumnMeta(mixedEqualStats, 177L));
+ assertFalse(canDrop(notEq(doubleColumn, 5.0), mixedEqualMetas));
+
+ // NaN literal: NaN values are present so cannot drop
+ assertFalse(canDrop(eq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(notEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(lt(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(ltEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(gt(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(gtEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(in(doubleColumn, new HashSet<>(List.of(Double.NaN))), metas));
+ }
+
+ @Test
+ public void testNaNDoubleMissingNaNCountIsConservative() {
+ org.apache.parquet.column.statistics.Statistics<?> statsWithUnknownNaNs =
+ org.apache.parquet.column.statistics.Statistics.getBuilderForReading(
+ Types.required(PrimitiveTypeName.DOUBLE).named("test_double"))
+ .withMin(BytesUtils.longToBytes(Double.doubleToLongBits(10.0)))
+ .withMax(BytesUtils.longToBytes(Double.doubleToLongBits(100.0)))
+ .withNumNulls(0)
+ .build();
+
+ List<ColumnChunkMetaData> metas =
+ List.of(getIntColumnMeta(intStats, 177L), getDoubleColumnMeta(statsWithUnknownNaNs, 177L));
+
+ assertFalse(canDrop(eq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(notEq(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(lt(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(ltEq(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(gt(doubleColumn, 200.0), metas));
+ assertFalse(canDrop(gtEq(doubleColumn, 200.0), metas));
+ }
+
+ @Test
+ public void testNaNDoubleZeroNaNCount() {
+ // Explicit zero NaN count with valid min/max
+ DoubleStatistics zeroNanStats = new DoubleStatistics();
+ zeroNanStats.setMinMax(10, 100);
+ zeroNanStats.setNumNulls(0);
+ zeroNanStats.incrementNanCount(0);
+
+ List<ColumnChunkMetaData> metas =
+ List.of(getIntColumnMeta(intStats, 177L), getDoubleColumnMeta(zeroNanStats, 177L));
+
+ assertFalse(canDrop(eq(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(notEq(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(lt(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(ltEq(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(gt(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(gtEq(doubleColumn, 50.0), metas));
+ assertFalse(canDrop(in(doubleColumn, new HashSet<>(List.of(50.0))), metas));
+
+ assertTrue(canDrop(eq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(notEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(lt(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(ltEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(gt(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(gtEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(in(doubleColumn, new HashSet<>(List.of(Double.NaN))), metas));
+ }
+
+ @Test
+ public void testNaNDoubleIeee754TotalOrder() {
+ // All-NaN with IEEE_754_TOTAL_ORDER, stats built via builder (no min/max)
+ PrimitiveType ieee754Type = Types.required(PrimitiveTypeName.DOUBLE)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("test_double");
+ org.apache.parquet.column.statistics.Statistics<?> allNanStats =
+ org.apache.parquet.column.statistics.Statistics.getBuilderForReading(ieee754Type)
+ .withNumNulls(0)
+ .withNanCount(177)
+ .build();
+
+ List<ColumnChunkMetaData> metas =
+ List.of(getIntColumnMeta(intStats, 177L), getDoubleColumnMetaWithType(ieee754Type, allNanStats, 177L));
+
+ assertTrue(canDrop(eq(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(notEq(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(lt(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(ltEq(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(gt(doubleColumn, 5.0), metas));
+ assertFalse(canDrop(gtEq(doubleColumn, 5.0), metas));
+ assertTrue(canDrop(in(doubleColumn, new HashSet<>(List.of(5.0))), metas));
+
+ assertFalse(canDrop(eq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(notEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(lt(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(ltEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(gt(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(gtEq(doubleColumn, Double.NaN), metas));
+ assertFalse(canDrop(in(doubleColumn, new HashSet<>(List.of(Double.NaN))), metas));
+ }
+
+ // ========================= Float NaN Tests =========================
+
+ @Test
+ public void testNaNFloatAllNaN() {
+ // All non-null values are NaN
+ FloatStatistics allNanStats = new FloatStatistics();
+ allNanStats.setNumNulls(0);
+ allNanStats.incrementNanCount(177);
+
+ List<ColumnChunkMetaData> metas =
+ List.of(getIntColumnMeta(intStats, 177L), getFloatColumnMeta(allNanStats, 177L));
+
+ assertTrue(canDrop(eq(floatCol, 5.0f), metas));
+ assertFalse(canDrop(notEq(floatCol, 5.0f), metas));
+ assertFalse(canDrop(lt(floatCol, 5.0f), metas));
+ assertFalse(canDrop(ltEq(floatCol, 5.0f), metas));
+ assertFalse(canDrop(gt(floatCol, 5.0f), metas));
+ assertFalse(canDrop(gtEq(floatCol, 5.0f), metas));
+ assertTrue(canDrop(in(floatCol, new HashSet<>(List.of(5.0f))), metas));
+
+ assertFalse(canDrop(eq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(notEq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(lt(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(ltEq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(gt(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(gtEq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(in(floatCol, new HashSet<>(List.of(Float.NaN))), metas));
+ }
+
+ @Test
+ public void testNaNFloatMixed() {
+ // Mixed: nan_count=50, min=10, max=100
+ FloatStatistics mixedStats = new FloatStatistics();
+ mixedStats.setMinMax(10.0f, 100.0f);
+ mixedStats.setNumNulls(0);
+ mixedStats.incrementNanCount(50);
+
+ List<ColumnChunkMetaData> metas =
+ List.of(getIntColumnMeta(intStats, 177L), getFloatColumnMeta(mixedStats, 177L));
+
+ assertFalse(canDrop(eq(floatCol, 50.0f), metas));
+ assertFalse(canDrop(notEq(floatCol, 50.0f), metas));
+ assertFalse(canDrop(lt(floatCol, 50.0f), metas));
+ assertFalse(canDrop(ltEq(floatCol, 50.0f), metas));
+ assertFalse(canDrop(gt(floatCol, 50.0f), metas));
+ assertFalse(canDrop(gtEq(floatCol, 50.0f), metas));
+ assertFalse(canDrop(in(floatCol, new HashSet<>(List.of(50.0f))), metas));
+ assertFalse(canDrop(lt(floatCol, 0.0f), metas));
+ assertFalse(canDrop(gt(floatCol, 200.0f), metas));
+
+ FloatStatistics mixedEqualStats = new FloatStatistics();
+ mixedEqualStats.setMinMax(5.0f, 5.0f);
+ mixedEqualStats.setNumNulls(0);
+ mixedEqualStats.incrementNanCount(1);
+ List<ColumnChunkMetaData> mixedEqualMetas =
+ List.of(getIntColumnMeta(intStats, 177L), getFloatColumnMeta(mixedEqualStats, 177L));
+ assertFalse(canDrop(notEq(floatCol, 5.0f), mixedEqualMetas));
+
+ assertFalse(canDrop(eq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(notEq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(lt(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(ltEq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(gt(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(gtEq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(in(floatCol, new HashSet<>(List.of(Float.NaN))), metas));
+ }
+
+ @Test
+ public void testNaNFloatZeroNaNCount() {
+ // Zero NaN count with valid min/max
+ FloatStatistics zeroNanStats = new FloatStatistics();
+ zeroNanStats.setMinMax(10.0f, 100.0f);
+ zeroNanStats.setNumNulls(0);
+ zeroNanStats.incrementNanCount(0);
+
+ List<ColumnChunkMetaData> metas =
+ List.of(getIntColumnMeta(intStats, 177L), getFloatColumnMeta(zeroNanStats, 177L));
+
+ assertFalse(canDrop(eq(floatCol, 50.0f), metas));
+ assertFalse(canDrop(notEq(floatCol, 50.0f), metas));
+ assertFalse(canDrop(lt(floatCol, 50.0f), metas));
+ assertFalse(canDrop(ltEq(floatCol, 50.0f), metas));
+ assertFalse(canDrop(gt(floatCol, 50.0f), metas));
+ assertFalse(canDrop(gtEq(floatCol, 50.0f), metas));
+ assertFalse(canDrop(in(floatCol, new HashSet<>(List.of(50.0f))), metas));
+
+ assertTrue(canDrop(eq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(notEq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(lt(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(ltEq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(gt(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(gtEq(floatCol, Float.NaN), metas));
+ assertFalse(canDrop(in(floatCol, new HashSet<>(List.of(Float.NaN))), metas));
+ }
+
+ // ========================= Float16 NaN Tests =========================
+
+ private static final PrimitiveType FLOAT16_TYPE = Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(LogicalTypeAnnotation.float16Type())
+ .named("test_float16");
+
+ private static final Binary FLOAT16_NAN = Binary.fromConstantByteArray(new byte[] {0x00, 0x7e});
+ private static final Binary FLOAT16_ONE = Binary.fromConstantByteArray(new byte[] {0x00, 0x3c});
+ private static final Binary FLOAT16_TEN = Binary.fromConstantByteArray(new byte[] {0x00, 0x49});
+ private static final Binary FLOAT16_HUNDRED = Binary.fromConstantByteArray(new byte[] {0x40, 0x56});
+ private static final Binary FLOAT16_FIFTY = Binary.fromConstantByteArray(new byte[] {0x40, 0x52});
+
+ private static final Operators.BinaryColumn float16Column = binaryColumn("float16.column");
+
+ private static ColumnChunkMetaData getFloat16ColumnMeta(
+ org.apache.parquet.column.statistics.Statistics<?> stats, long valueCount) {
+ return ColumnChunkMetaData.get(
+ ColumnPath.get("float16", "column"),
+ FLOAT16_TYPE,
+ CompressionCodecName.GZIP,
+ null,
+ new HashSet<>(List.of(Encoding.PLAIN)),
+ stats,
+ 0L,
+ 0L,
+ valueCount,
+ 0L,
+ 0L);
+ }
+
+ @Test
+ public void testNaNFloat16AllNaN() {
+ // All non-null values are NaN
+ org.apache.parquet.column.statistics.Statistics<?> allNanStats =
+ org.apache.parquet.column.statistics.Statistics.getBuilderForReading(FLOAT16_TYPE)
+ .withNumNulls(0)
+ .withNanCount(177)
+ .build();
+
+ ColumnChunkMetaData float16Meta = getFloat16ColumnMeta(allNanStats, 177L);
+ List<ColumnChunkMetaData> metas = List.of(getIntColumnMeta(intStats, 177L), float16Meta);
+
+ assertTrue(canDrop(eq(float16Column, FLOAT16_ONE), metas));
+ assertFalse(canDrop(notEq(float16Column, FLOAT16_ONE), metas));
+ assertFalse(canDrop(lt(float16Column, FLOAT16_ONE), metas));
+ assertFalse(canDrop(ltEq(float16Column, FLOAT16_ONE), metas));
+ assertFalse(canDrop(gt(float16Column, FLOAT16_ONE), metas));
+ assertFalse(canDrop(gtEq(float16Column, FLOAT16_ONE), metas));
+ assertTrue(canDrop(in(float16Column, new HashSet<>(List.of(FLOAT16_ONE))), metas));
+
+ assertFalse(canDrop(eq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(notEq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(lt(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(ltEq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(gt(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(gtEq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(in(float16Column, new HashSet<>(List.of(FLOAT16_NAN))), metas));
+ }
+
+ @Test
+ public void testNaNFloat16Mixed() {
+ // Mixed: nan_count=50, min=10, max=100
+ org.apache.parquet.column.statistics.Statistics<?> mixedStats =
+ org.apache.parquet.column.statistics.Statistics.getBuilderForReading(FLOAT16_TYPE)
+ .withMin(FLOAT16_TEN.getBytes())
+ .withMax(FLOAT16_HUNDRED.getBytes())
+ .withNumNulls(0)
+ .withNanCount(50)
+ .build();
+
+ ColumnChunkMetaData float16Meta = getFloat16ColumnMeta(mixedStats, 177L);
+ List<ColumnChunkMetaData> metas = List.of(getIntColumnMeta(intStats, 177L), float16Meta);
+
+ assertFalse(canDrop(eq(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(notEq(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(lt(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(ltEq(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(gt(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(gtEq(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(in(float16Column, new HashSet<>(List.of(FLOAT16_FIFTY))), metas));
+
+ assertFalse(canDrop(eq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(notEq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(lt(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(ltEq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(gt(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(gtEq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(in(float16Column, new HashSet<>(List.of(FLOAT16_NAN))), metas));
+ }
+
+ @Test
+ public void testNaNFloat16ZeroNaNCount() {
+ // Zero NaN count with valid min/max
+ org.apache.parquet.column.statistics.Statistics<?> zeroNanStats =
+ org.apache.parquet.column.statistics.Statistics.getBuilderForReading(FLOAT16_TYPE)
+ .withMin(FLOAT16_TEN.getBytes())
+ .withMax(FLOAT16_HUNDRED.getBytes())
+ .withNumNulls(0)
+ .withNanCount(0)
+ .build();
+
+ ColumnChunkMetaData float16Meta = getFloat16ColumnMeta(zeroNanStats, 177L);
+ List<ColumnChunkMetaData> metas = List.of(getIntColumnMeta(intStats, 177L), float16Meta);
+
+ assertFalse(canDrop(eq(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(notEq(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(lt(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(ltEq(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(gt(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(gtEq(float16Column, FLOAT16_FIFTY), metas));
+ assertFalse(canDrop(in(float16Column, new HashSet<>(List.of(FLOAT16_FIFTY))), metas));
+
+ assertTrue(canDrop(eq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(notEq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(lt(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(ltEq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(gt(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(gtEq(float16Column, FLOAT16_NAN), metas));
+ assertFalse(canDrop(in(float16Column, new HashSet<>(List.of(FLOAT16_NAN))), metas));
+ }
}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
index 7972a75ba..8d778f7b9 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -1978,4 +1978,185 @@ public class TestParquetMetadataConverter {
assertNull(ParquetMetadataConverter.fromParquetEdgeInterpolationAlgorithm(null));
assertNull(ParquetMetadataConverter.toParquetEdgeInterpolationAlgorithm(null));
}
+
+ @Test
+ public void testIEEE754TotalOrderColumnOrder() throws IOException {
+ MessageType schema = Types.buildMessage()
+ .required(PrimitiveTypeName.FLOAT)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("float_ieee754")
+ .required(PrimitiveTypeName.DOUBLE)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("double_ieee754")
+ .named("Message");
+
+ org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData =
+ new org.apache.parquet.hadoop.metadata.FileMetaData(schema, new HashMap<String, String>(), null);
+ ParquetMetadata metadata = new ParquetMetadata(fileMetaData, new ArrayList<BlockMetaData>());
+ ParquetMetadataConverter converter = new ParquetMetadataConverter();
+ FileMetaData formatMetadata = converter.toParquetMetadata(1, metadata);
+
+ List<org.apache.parquet.format.ColumnOrder> columnOrders = formatMetadata.getColumn_orders();
+ assertEquals(2, columnOrders.size());
+ for (org.apache.parquet.format.ColumnOrder columnOrder : columnOrders) {
+ assertTrue(columnOrder.isSetIEEE_754_TOTAL_ORDER());
+ }
+
+ MessageType resultSchema =
+ converter.fromParquetMetadata(formatMetadata).getFileMetaData().getSchema();
+ assertEquals(
+ ColumnOrder.ieee754TotalOrder(),
+ resultSchema.getType("float_ieee754").asPrimitiveType().columnOrder());
+ assertEquals(
+ ColumnOrder.ieee754TotalOrder(),
+ resultSchema.getType("double_ieee754").asPrimitiveType().columnOrder());
+ }
+
+ @Test
+ public void testNestedColumnOrdersUseLeafOrder() throws IOException {
+ MessageType schema = Types.buildMessage()
+ .requiredGroup()
+ .required(PrimitiveTypeName.FLOAT)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("a")
+ .required(PrimitiveTypeName.DOUBLE)
+ .named("b")
+ .named("g")
+ .required(PrimitiveTypeName.DOUBLE)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("c")
+ .named("Message");
+
+ org.apache.parquet.hadoop.metadata.FileMetaData fileMetaData =
+ new org.apache.parquet.hadoop.metadata.FileMetaData(schema, new HashMap<String, String>(), null);
+ ParquetMetadata metadata = new ParquetMetadata(fileMetaData, new ArrayList<BlockMetaData>());
+ ParquetMetadataConverter converter = new ParquetMetadataConverter();
+ FileMetaData formatMetadata = converter.toParquetMetadata(1, metadata);
+
+ MessageType resultSchema =
+ converter.fromParquetMetadata(formatMetadata).getFileMetaData().getSchema();
+ List<ColumnDescriptor> columns = resultSchema.getColumns();
+ assertEquals(3, columns.size());
+ assertEquals(
+ ColumnOrder.ieee754TotalOrder(),
+ columns.get(0).getPrimitiveType().columnOrder());
+ assertEquals(
+ ColumnOrder.typeDefined(), columns.get(1).getPrimitiveType().columnOrder());
+ assertEquals(
+ ColumnOrder.ieee754TotalOrder(),
+ columns.get(2).getPrimitiveType().columnOrder());
+ }
+
+ @Test
+ public void testStatisticsNanCountRoundTripFloat() {
+ PrimitiveType type = Types.required(PrimitiveTypeName.FLOAT).named("test_float");
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(type);
+ stats.updateStats(1.0f);
+ stats.updateStats(Float.NaN);
+ stats.updateStats(3.0f);
+ stats.updateStats(Float.NaN);
+
+ org.apache.parquet.format.Statistics formatStats = ParquetMetadataConverter.toParquetStatistics(stats);
+ assertTrue("nan_count should be set", formatStats.isSetNan_count());
+ assertEquals(2, formatStats.getNan_count());
+
+ Statistics<?> roundTrip = ParquetMetadataConverter.fromParquetStatisticsInternal(
+ Version.FULL_VERSION, formatStats, type, ParquetMetadataConverter.SortOrder.SIGNED);
+ assertTrue(roundTrip.isNanCountSet());
+ assertEquals(2, roundTrip.getNanCount());
+ }
+
+ @Test
+ public void testStatisticsNanCountRoundTripDouble() {
+ PrimitiveType type = Types.required(PrimitiveTypeName.DOUBLE).named("test_double");
+ DoubleStatistics stats = (DoubleStatistics) Statistics.createStats(type);
+ stats.updateStats(1.0);
+ stats.updateStats(Double.NaN);
+ stats.updateStats(3.0);
+
+ org.apache.parquet.format.Statistics formatStats = ParquetMetadataConverter.toParquetStatistics(stats);
+ assertTrue("nan_count should be set", formatStats.isSetNan_count());
+ assertEquals(1, formatStats.getNan_count());
+
+ Statistics<?> roundTrip = ParquetMetadataConverter.fromParquetStatisticsInternal(
+ Version.FULL_VERSION, formatStats, type, ParquetMetadataConverter.SortOrder.SIGNED);
+ assertTrue(roundTrip.isNanCountSet());
+ assertEquals(1, roundTrip.getNanCount());
+ }
+
+ @Test
+ public void testStatisticsNanCountRoundTripFloat16() {
+ PrimitiveType type = Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(LogicalTypeAnnotation.float16Type())
+ .named("test_float16");
+ BinaryStatistics stats = (BinaryStatistics) Statistics.createStats(type);
+ // FLOAT16 1.0 = 0x3C00
+ stats.updateStats(Binary.fromConstantByteArray(new byte[] {0x00, 0x3C}));
+ // FLOAT16 NaN = 0x7E00
+ stats.updateStats(Binary.fromConstantByteArray(new byte[] {0x00, 0x7E}));
+
+ org.apache.parquet.format.Statistics formatStats = ParquetMetadataConverter.toParquetStatistics(stats);
+ assertTrue("nan_count should be set", formatStats.isSetNan_count());
+ assertEquals(1, formatStats.getNan_count());
+
+ Statistics<?> roundTrip = ParquetMetadataConverter.fromParquetStatisticsInternal(
+ Version.FULL_VERSION, formatStats, type, ParquetMetadataConverter.SortOrder.SIGNED);
+ assertTrue(roundTrip.isNanCountSet());
+ assertEquals(1, roundTrip.getNanCount());
+ }
+
+ @Test
+ public void testStatisticsNanCountZeroRoundTrip() {
+ PrimitiveType type = Types.required(PrimitiveTypeName.FLOAT).named("test_float");
+ FloatStatistics stats = (FloatStatistics) Statistics.createStats(type);
+ stats.updateStats(1.0f);
+ stats.updateStats(2.0f);
+
+ org.apache.parquet.format.Statistics formatStats = ParquetMetadataConverter.toParquetStatistics(stats);
+ assertTrue("nan_count should be set even when zero", formatStats.isSetNan_count());
+ assertEquals(0, formatStats.getNan_count());
+
+ Statistics<?> roundTrip = ParquetMetadataConverter.fromParquetStatisticsInternal(
+ Version.FULL_VERSION, formatStats, type, ParquetMetadataConverter.SortOrder.SIGNED);
+ assertTrue(roundTrip.isNanCountSet());
+ assertEquals(0, roundTrip.getNanCount());
+ }
+
+ @Test
+ public void testColumnIndexNanCountsRoundTrip() {
+ PrimitiveType type = Types.required(PrimitiveTypeName.FLOAT)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("test_float");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
+
+ // Page 1: mixed NaN and non-NaN
+ FloatStatistics stats1 = (FloatStatistics) Statistics.createStats(type);
+ stats1.updateStats(1.0f);
+ stats1.updateStats(Float.NaN);
+ stats1.updateStats(3.0f);
+ builder.add(stats1);
+
+ // Page 2: all nulls
+ FloatStatistics stats2 = (FloatStatistics) Statistics.createStats(type);
+ stats2.incrementNumNulls(10);
+ builder.add(stats2);
+
+ // Page 3: no NaN
+ FloatStatistics stats3 = (FloatStatistics) Statistics.createStats(type);
+ stats3.updateStats(5.0f);
+ stats3.updateStats(10.0f);
+ builder.add(stats3);
+
+ ColumnIndex columnIndex = builder.build();
+ org.apache.parquet.format.ColumnIndex parquetColumnIndex =
+ ParquetMetadataConverter.toParquetColumnIndex(type, columnIndex);
+ assertNotNull(parquetColumnIndex);
+ assertNotNull("nan_counts should be set", parquetColumnIndex.getNan_counts());
+ assertEquals(List.of(1L, 0L, 0L), parquetColumnIndex.getNan_counts());
+
+ ColumnIndex roundTrip = ParquetMetadataConverter.fromParquetColumnIndex(type, parquetColumnIndex);
+ assertNotNull(roundTrip);
+ assertEquals(List.of(1L, 0L, 0L), roundTrip.getNanCounts());
+ }
}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/FloatingPointNanInteropFileGenerator.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/FloatingPointNanInteropFileGenerator.java
new file mode 100644
index 000000000..df4b8ce88
--- /dev/null
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/FloatingPointNanInteropFileGenerator.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.hadoop;
+
+import static org.apache.parquet.schema.LogicalTypeAnnotation.float16Type;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
+import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.GroupFactory;
+import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.hadoop.rewrite.ParquetRewriter;
+import org.apache.parquet.hadoop.rewrite.RewriteOptions;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.Types;
+
+public final class FloatingPointNanInteropFileGenerator {
+
+ public static final String FILE_NO_NAN = "floating_orders_nan_count_no_nan.parquet";
+ public static final String FILE_MIXED_NAN = "floating_orders_nan_count_mixed_nan.parquet";
+ public static final String FILE_ALL_NAN = "floating_orders_nan_count_all_nan.parquet";
+ public static final String FILE_ZERO_MIN = "floating_orders_nan_count_zero_min.parquet";
+ public static final String FILE_ZERO_MAX = "floating_orders_nan_count_zero_max.parquet";
+ public static final String FILE_MERGED = "floating_orders_nan_count_merged.parquet";
+
+ public static final int ROWS_PER_FILE = 10;
+
+ private static final float FLOAT_NEG_NAN_SMALL = Float.intBitsToFloat(0xffffffff);
+ private static final float FLOAT_NEG_NAN_LARGE = Float.intBitsToFloat(0xfff00001);
+ private static final float FLOAT_NAN_SMALL = Float.intBitsToFloat(0x7fc00001);
+ private static final float FLOAT_NAN_LARGE = Float.intBitsToFloat(0x7fffffff);
+ private static final double DOUBLE_NEG_NAN_SMALL = Double.longBitsToDouble(0xffffffffffffffffL);
+ private static final double DOUBLE_NEG_NAN_LARGE = Double.longBitsToDouble(0xfff0000000000001L);
+ private static final double DOUBLE_NAN_SMALL = Double.longBitsToDouble(0x7ff0000000000001L);
+ private static final double DOUBLE_NAN_LARGE = Double.longBitsToDouble(0x7fffffffffffffffL);
+ private static final Binary FLOAT16_NEG_NAN_SMALL = float16Binary((short) 0xffff);
+ private static final Binary FLOAT16_NEG_NAN_LARGE = float16Binary((short) 0xfc01);
+ private static final Binary FLOAT16_NAN_SMALL = float16Binary((short) 0x7c01);
+ private static final Binary FLOAT16_NAN_LARGE = float16Binary((short) 0x7fff);
+
+ private static final MessageType SCHEMA = Types.buildMessage()
+ .required(FLOAT)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("float_ieee754")
+ .required(FLOAT)
+ .named("float_typedef")
+ .required(DOUBLE)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("double_ieee754")
+ .required(DOUBLE)
+ .named("double_typedef")
+ .required(FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(float16Type())
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("float16_ieee754")
+ .required(FIXED_LEN_BYTE_ARRAY)
+ .length(2)
+ .as(float16Type())
+ .named("float16_typedef")
+ .named("msg");
+
+ private static final float[] NO_NAN_FLOATS = new float[] {-2f, -1f, -0f, 0f, 0.5f, 1f, 2f, 3f, 4f, 5f};
+ private static final double[] NO_NAN_DOUBLES = new double[] {-2d, -1d, -0d, 0d, 0.5d, 1d, 2d, 3d, 4d, 5d};
+ private static final Binary[] NO_NAN_FLOAT16 = new Binary[] {
+ float16Binary((short) 0xc000),
+ float16Binary((short) 0xbc00),
+ float16Binary((short) 0x8000),
+ float16Binary((short) 0x0000),
+ float16Binary((short) 0x3800),
+ float16Binary((short) 0x3c00),
+ float16Binary((short) 0x4000),
+ float16Binary((short) 0x4200),
+ float16Binary((short) 0x4400),
+ float16Binary((short) 0x4500)
+ };
+
+ private static final float[] MIXED_NAN_FLOATS = new float[] {
+ FLOAT_NEG_NAN_SMALL, -2f, FLOAT_NEG_NAN_LARGE, -1f, -0f, 0f, 1f, FLOAT_NAN_SMALL, 3f, FLOAT_NAN_LARGE
+ };
+ private static final double[] MIXED_NAN_DOUBLES = new double[] {
+ DOUBLE_NEG_NAN_SMALL, -2d, DOUBLE_NEG_NAN_LARGE, -1d, -0d, 0d, 1d, DOUBLE_NAN_SMALL, 3d, DOUBLE_NAN_LARGE
+ };
+ private static final Binary[] MIXED_NAN_FLOAT16 = new Binary[] {
+ FLOAT16_NEG_NAN_SMALL,
+ float16Binary((short) 0xc000),
+ FLOAT16_NEG_NAN_LARGE,
+ float16Binary((short) 0xbc00),
+ float16Binary((short) 0x8000),
+ float16Binary((short) 0x0000),
+ float16Binary((short) 0x3c00),
+ FLOAT16_NAN_SMALL,
+ float16Binary((short) 0x4200),
+ FLOAT16_NAN_LARGE
+ };
+
+ private static final float[] ALL_NAN_FLOATS = new float[] {
+ FLOAT_NEG_NAN_SMALL,
+ FLOAT_NEG_NAN_LARGE,
+ FLOAT_NAN_SMALL,
+ FLOAT_NAN_LARGE,
+ FLOAT_NEG_NAN_SMALL,
+ FLOAT_NEG_NAN_LARGE,
+ FLOAT_NAN_SMALL,
+ FLOAT_NAN_LARGE,
+ FLOAT_NEG_NAN_SMALL,
+ FLOAT_NAN_LARGE
+ };
+
+ private static final double[] ALL_NAN_DOUBLES = new double[] {
+ DOUBLE_NEG_NAN_SMALL,
+ DOUBLE_NEG_NAN_LARGE,
+ DOUBLE_NAN_SMALL,
+ DOUBLE_NAN_LARGE,
+ DOUBLE_NEG_NAN_SMALL,
+ DOUBLE_NEG_NAN_LARGE,
+ DOUBLE_NAN_SMALL,
+ DOUBLE_NAN_LARGE,
+ DOUBLE_NEG_NAN_SMALL,
+ DOUBLE_NAN_LARGE
+ };
+
+ private static final Binary[] ALL_NAN_FLOAT16 = new Binary[] {
+ FLOAT16_NEG_NAN_SMALL,
+ FLOAT16_NEG_NAN_LARGE,
+ FLOAT16_NAN_SMALL,
+ FLOAT16_NAN_LARGE,
+ FLOAT16_NEG_NAN_SMALL,
+ FLOAT16_NEG_NAN_LARGE,
+ FLOAT16_NAN_SMALL,
+ FLOAT16_NAN_LARGE,
+ FLOAT16_NEG_NAN_SMALL,
+ FLOAT16_NAN_LARGE
+ };
+
+ private static final float[] ZERO_MIN_FLOATS = new float[] {0f, 0f, 0f, 0.5f, 1f, 1.5f, 2f, 3f, 4f, 5f};
+ private static final double[] ZERO_MIN_DOUBLES = new double[] {0d, 0d, 0d, 0.5d, 1d, 1.5d, 2d, 3d, 4d, 5d};
+ private static final Binary[] ZERO_MIN_FLOAT16 = new Binary[] {
+ float16Binary((short) 0x0000),
+ float16Binary((short) 0x0000),
+ float16Binary((short) 0x0000),
+ float16Binary((short) 0x3800),
+ float16Binary((short) 0x3c00),
+ float16Binary((short) 0x3e00),
+ float16Binary((short) 0x4000),
+ float16Binary((short) 0x4200),
+ float16Binary((short) 0x4400),
+ float16Binary((short) 0x4500)
+ };
+
+ private static final float[] ZERO_MAX_FLOATS = new float[] {-5f, -4f, -3f, -2f, -1.5f, -1f, -0.5f, -0f, -0f, -0f};
+ private static final double[] ZERO_MAX_DOUBLES =
+ new double[] {-5d, -4d, -3d, -2d, -1.5d, -1d, -0.5d, -0d, -0d, -0d};
+ private static final Binary[] ZERO_MAX_FLOAT16 = new Binary[] {
+ float16Binary((short) 0xc500),
+ float16Binary((short) 0xc400),
+ float16Binary((short) 0xc200),
+ float16Binary((short) 0xc000),
+ float16Binary((short) 0xbe00),
+ float16Binary((short) 0xbc00),
+ float16Binary((short) 0xb800),
+ float16Binary((short) 0x8000),
+ float16Binary((short) 0x8000),
+ float16Binary((short) 0x8000)
+ };
+
+ public enum Scenario {
+ NO_NAN,
+ MIXED_NAN,
+ ALL_NAN,
+ ZERO_MIN,
+ ZERO_MAX
+ }
+
+ public static final class GenerationResult {
+ private final Path noNanFile;
+ private final Path mixedNanFile;
+ private final Path allNanFile;
+ private final Path zeroMinFile;
+ private final Path zeroMaxFile;
+ private final Path mergedFile;
+
+ private GenerationResult(
+ Path noNanFile,
+ Path mixedNanFile,
+ Path allNanFile,
+ Path zeroMinFile,
+ Path zeroMaxFile,
+ Path mergedFile) {
+ this.noNanFile = noNanFile;
+ this.mixedNanFile = mixedNanFile;
+ this.allNanFile = allNanFile;
+ this.zeroMinFile = zeroMinFile;
+ this.zeroMaxFile = zeroMaxFile;
+ this.mergedFile = mergedFile;
+ }
+
+ public Path getNoNanFile() {
+ return noNanFile;
+ }
+
+ public Path getMixedNanFile() {
+ return mixedNanFile;
+ }
+
+ public Path getAllNanFile() {
+ return allNanFile;
+ }
+
+ public Path getZeroMinFile() {
+ return zeroMinFile;
+ }
+
+ public Path getZeroMaxFile() {
+ return zeroMaxFile;
+ }
+
+ public Path getMergedFile() {
+ return mergedFile;
+ }
+ }
+
+ private FloatingPointNanInteropFileGenerator() {}
+
+ public static MessageType schema() {
+ return SCHEMA;
+ }
+
+ public static float[] floatValues(Scenario scenario) {
+ switch (scenario) {
+ case NO_NAN:
+ return Arrays.copyOf(NO_NAN_FLOATS, NO_NAN_FLOATS.length);
+ case MIXED_NAN:
+ return Arrays.copyOf(MIXED_NAN_FLOATS, MIXED_NAN_FLOATS.length);
+ case ALL_NAN:
+ return Arrays.copyOf(ALL_NAN_FLOATS, ALL_NAN_FLOATS.length);
+ case ZERO_MIN:
+ return Arrays.copyOf(ZERO_MIN_FLOATS, ZERO_MIN_FLOATS.length);
+ case ZERO_MAX:
+ return Arrays.copyOf(ZERO_MAX_FLOATS, ZERO_MAX_FLOATS.length);
+ default:
+ throw new IllegalArgumentException("Unknown scenario: " + scenario);
+ }
+ }
+
+ public static double[] doubleValues(Scenario scenario) {
+ switch (scenario) {
+ case NO_NAN:
+ return Arrays.copyOf(NO_NAN_DOUBLES, NO_NAN_DOUBLES.length);
+ case MIXED_NAN:
+ return Arrays.copyOf(MIXED_NAN_DOUBLES, MIXED_NAN_DOUBLES.length);
+ case ALL_NAN:
+ return Arrays.copyOf(ALL_NAN_DOUBLES, ALL_NAN_DOUBLES.length);
+ case ZERO_MIN:
+ return Arrays.copyOf(ZERO_MIN_DOUBLES, ZERO_MIN_DOUBLES.length);
+ case ZERO_MAX:
+ return Arrays.copyOf(ZERO_MAX_DOUBLES, ZERO_MAX_DOUBLES.length);
+ default:
+ throw new IllegalArgumentException("Unknown scenario: " + scenario);
+ }
+ }
+
+ public static Binary[] float16Values(Scenario scenario) {
+ Binary[] values;
+ switch (scenario) {
+ case NO_NAN:
+ values = NO_NAN_FLOAT16;
+ break;
+ case MIXED_NAN:
+ values = MIXED_NAN_FLOAT16;
+ break;
+ case ALL_NAN:
+ values = ALL_NAN_FLOAT16;
+ break;
+ case ZERO_MIN:
+ values = ZERO_MIN_FLOAT16;
+ break;
+ case ZERO_MAX:
+ values = ZERO_MAX_FLOAT16;
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown scenario: " + scenario);
+ }
+
+ Binary[] copy = new Binary[values.length];
+ for (int i = 0; i < values.length; i++) {
+ copy[i] = values[i].copy();
+ }
+ return copy;
+ }
+
+ public static GenerationResult generateAndMerge(Configuration conf, Path outputDir) throws IOException {
+ FileSystem fs = outputDir.getFileSystem(conf);
+ if (!fs.exists(outputDir) && !fs.mkdirs(outputDir)) {
+ throw new IOException("Cannot create output directory: " + outputDir);
+ }
+
+ Path noNanPath = new Path(outputDir, FILE_NO_NAN);
+ Path mixedNanPath = new Path(outputDir, FILE_MIXED_NAN);
+ Path allNanPath = new Path(outputDir, FILE_ALL_NAN);
+ Path zeroMinPath = new Path(outputDir, FILE_ZERO_MIN);
+ Path zeroMaxPath = new Path(outputDir, FILE_ZERO_MAX);
+ Path mergedPath = new Path(outputDir, FILE_MERGED);
+
+ deleteIfExists(fs, noNanPath);
+ deleteIfExists(fs, mixedNanPath);
+ deleteIfExists(fs, allNanPath);
+ deleteIfExists(fs, zeroMinPath);
+ deleteIfExists(fs, zeroMaxPath);
+ deleteIfExists(fs, mergedPath);
+
+ writeScenarioFile(conf, noNanPath, Scenario.NO_NAN);
+ writeScenarioFile(conf, mixedNanPath, Scenario.MIXED_NAN);
+ writeScenarioFile(conf, allNanPath, Scenario.ALL_NAN);
+ writeScenarioFile(conf, zeroMinPath, Scenario.ZERO_MIN);
+ writeScenarioFile(conf, zeroMaxPath, Scenario.ZERO_MAX);
+ mergeFiles(conf, List.of(noNanPath, mixedNanPath, allNanPath, zeroMinPath, zeroMaxPath), mergedPath);
+
+ return new GenerationResult(noNanPath, mixedNanPath, allNanPath, zeroMinPath, zeroMaxPath, mergedPath);
+ }
+
+ public static void writeScenarioFile(Configuration conf, Path outputFile, Scenario scenario) throws IOException {
+ float[] floatValues = floatValues(scenario);
+ double[] doubleValues = doubleValues(scenario);
+ Binary[] float16Values = float16Values(scenario);
+
+ if (floatValues.length != ROWS_PER_FILE
+ || doubleValues.length != ROWS_PER_FILE
+ || float16Values.length != ROWS_PER_FILE) {
+ throw new IllegalStateException("Scenario " + scenario + " must have exactly " + ROWS_PER_FILE + " rows");
+ }
+
+ GroupFactory factory = new SimpleGroupFactory(SCHEMA);
+ try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(outputFile)
+ .withConf(conf)
+ .withType(SCHEMA)
+ .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+ .withDictionaryEncoding(false)
+ .build()) {
+ for (int i = 0; i < ROWS_PER_FILE; i++) {
+ writer.write(factory.newGroup()
+ .append("float_ieee754", floatValues[i])
+ .append("float_typedef", floatValues[i])
+ .append("double_ieee754", doubleValues[i])
+ .append("double_typedef", doubleValues[i])
+ .append("float16_ieee754", float16Values[i])
+ .append("float16_typedef", float16Values[i]));
+ }
+ }
+ }
+
+ public static void mergeFiles(Configuration conf, List<Path> inputFiles, Path outputFile) throws IOException {
+ RewriteOptions options = new RewriteOptions.Builder(conf, inputFiles, outputFile)
+ .transform(CompressionCodecName.UNCOMPRESSED)
+ .build();
+ try (ParquetRewriter rewriter = new ParquetRewriter(options)) {
+ rewriter.processBlocks();
+ }
+ }
+
+ private static void deleteIfExists(FileSystem fs, Path path) throws IOException {
+ if (fs.exists(path) && !fs.delete(path, false)) {
+ throw new IOException("Failed to delete existing file: " + path);
+ }
+ }
+
+ private static Binary float16Binary(short bits) {
+ return Binary.fromConstantByteArray(new byte[] {(byte) (bits & 0xff), (byte) ((bits >> 8) & 0xff)});
+ }
+
+ public static void main(String[] args) throws IOException {
+ String outputDir = args.length > 0 ? args[0] : "target/parquet-testing/data";
+ Configuration conf = new Configuration();
+ GenerationResult result = generateAndMerge(conf, new Path(outputDir));
+ System.out.println("Generated files:");
+ System.out.println(" " + result.getNoNanFile());
+ System.out.println(" " + result.getMixedNanFile());
+ System.out.println(" " + result.getAllNanFile());
+ System.out.println(" " + result.getZeroMinFile());
+ System.out.println(" " + result.getZeroMaxFile());
+ System.out.println(" " + result.getMergedFile());
+ }
+}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadFloatingPointNanCount.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadFloatingPointNanCount.java
new file mode 100644
index 000000000..52226b07c
--- /dev/null
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInterOpReadFloatingPointNanCount.java
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.hadoop;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.column.statistics.BinaryStatistics;
+import org.apache.parquet.column.statistics.DoubleStatistics;
+import org.apache.parquet.column.statistics.FloatStatistics;
+import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.hadoop.FloatingPointNanInteropFileGenerator.GenerationResult;
+import org.apache.parquet.hadoop.FloatingPointNanInteropFileGenerator.Scenario;
+import org.apache.parquet.hadoop.example.GroupReadSupport;
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.io.api.Binary;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class TestInterOpReadFloatingPointNanCount {
+
+ // Canonical parquet-testing fixture exercising the same scenarios as the local generator, as a
+ // single file with five row groups. parquet-java's writer produces the same statistics and
+ // column index layout, so the reference file is verified with the exact same expectations as the
+ // locally generated merged file.
+ private static final String REFERENCE_FILE = "floating_orders_nan_count.parquet";
+ private static final String REFERENCE_CHANGESET = "ffdcbb5e22828186c7461e56dbd26a0fe3caee56";
+
+ private final InterOpTester interop = new InterOpTester();
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ @Test
+ public void testReadStatsAndColumnIndex() throws IOException {
+ Configuration conf = new Configuration();
+ GenerationResult generated = FloatingPointNanInteropFileGenerator.generateAndMerge(
+ conf, new Path(temp.newFolder().getAbsolutePath()));
+
+ verifyFile(generated.getNoNanFile(), conf, Scenario.NO_NAN);
+ verifyFile(generated.getMixedNanFile(), conf, Scenario.MIXED_NAN);
+ verifyFile(generated.getAllNanFile(), conf, Scenario.ALL_NAN);
+ verifyFile(generated.getZeroMinFile(), conf, Scenario.ZERO_MIN);
+ verifyFile(generated.getZeroMaxFile(), conf, Scenario.ZERO_MAX);
+ verifyFile(
+ generated.getMergedFile(),
+ conf,
+ Scenario.NO_NAN,
+ Scenario.MIXED_NAN,
+ Scenario.ALL_NAN,
+ Scenario.ZERO_MIN,
+ Scenario.ZERO_MAX);
+ }
+
+ /**
+ * Reads the canonical {@code floating_orders_nan_count.parquet} from the parquet-testing repo and
+ * verifies it against the same expectations as the locally generated files, confirming that
+ * parquet-java reads the reference file (produced by another writer) consistently.
+ */
+ @Test
+ public void testReadReferenceFile() throws IOException {
+ Configuration conf = new Configuration();
+ Path file = interop.GetInterOpFile(REFERENCE_FILE, REFERENCE_CHANGESET);
+ verifyFile(
+ file,
+ conf,
+ Scenario.NO_NAN,
+ Scenario.MIXED_NAN,
+ Scenario.ALL_NAN,
+ Scenario.ZERO_MIN,
+ Scenario.ZERO_MAX);
+ }
+
+ private static void verifyFile(Path file, Configuration conf, Scenario... scenarios) throws IOException {
+ List<Float> expectFloats = concatFloatValues(scenarios);
+ List<Double> expectDoubles = concatDoubleValues(scenarios);
+ List<Binary> expectFloat16s = concatFloat16Values(scenarios);
+
+ try (ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(file, conf));
+ ParquetReader<Group> recordReader = ParquetReader.builder(new GroupReadSupport(), file)
+ .withConf(conf)
+ .build()) {
+ List<BlockMetaData> blocks = reader.getFooter().getBlocks();
+ assertEquals(scenarios.length, blocks.size());
+
+ // Verify read values
+ int rowId = 0;
+ Group group;
+ while ((group = recordReader.read()) != null) {
+ assertFloatValue(expectFloats.get(rowId), group.getFloat("float_ieee754", 0));
+ assertFloatValue(expectFloats.get(rowId), group.getFloat("float_typedef", 0));
+
+ assertDoubleValue(expectDoubles.get(rowId), group.getDouble("double_ieee754", 0));
+ assertDoubleValue(expectDoubles.get(rowId), group.getDouble("double_typedef", 0));
+
+ assertEquals(expectFloat16s.get(rowId), group.getBinary("float16_ieee754", 0));
+ assertEquals(expectFloat16s.get(rowId), group.getBinary("float16_typedef", 0));
+ rowId++;
+ }
+ assertEquals(10 * scenarios.length, rowId);
+
+ // Verify stats
+ for (int rg = 0; rg < blocks.size(); rg++) {
+ BlockMetaData block = blocks.get(rg);
+ assertEquals(10L, block.getRowCount());
+ for (ColumnChunkMetaData column : block.getColumns()) {
+ verifyStats(scenarios[rg], column);
+ verifyColumnIndex(reader, scenarios[rg], column);
+ }
+ }
+ }
+ }
+
+ private static void verifyStats(Scenario scenario, ColumnChunkMetaData column) {
+ String name = column.getPath().toDotString();
+ Statistics<?> stats = column.getStatistics();
+ assertTrue(stats.isNanCountSet());
+
+ if (name.contains("float16")) {
+ verifyFloat16Stats(scenario, name, (BinaryStatistics) stats);
+ } else if (name.contains("double")) {
+ verifyDoubleStats(scenario, name, (DoubleStatistics) stats);
+ } else {
+ verifyFloatStats(scenario, name, (FloatStatistics) stats);
+ }
+ }
+
+ private static void verifyFloatStats(Scenario scenario, String name, FloatStatistics stats) {
+ if (scenario == Scenario.NO_NAN) {
+ assertEquals(0L, stats.getNanCount());
+ assertEquals(-2f, stats.getMin(), 0f);
+ assertEquals(5f, stats.getMax(), 0f);
+ return;
+ }
+
+ if (scenario == Scenario.MIXED_NAN) {
+ assertEquals(4L, stats.getNanCount());
+ if (isIeee(name)) {
+ assertEquals(-2f, stats.getMin(), 0f);
+ assertEquals(3f, stats.getMax(), 0f);
+ }
+ return;
+ }
+
+ if (scenario == Scenario.ALL_NAN) {
+ assertEquals(10L, stats.getNanCount());
+ if (isIeee(name)) {
+ assertEquals(0xffffffff, Float.floatToRawIntBits(stats.getMin()));
+ assertEquals(0x7fffffff, Float.floatToRawIntBits(stats.getMax()));
+ }
+ return;
+ }
+
+ if (scenario == Scenario.ZERO_MIN) {
+ assertEquals(0L, stats.getNanCount());
+ assertEquals(isIeee(name) ? 0x00000000 : 0x80000000, Float.floatToRawIntBits(stats.getMin()));
+ assertEquals(5f, stats.getMax(), 0f);
+ return;
+ }
+
+ assertEquals(0L, stats.getNanCount());
+ assertEquals(-5f, stats.getMin(), 0f);
+ assertEquals(isIeee(name) ? 0x80000000 : 0x00000000, Float.floatToRawIntBits(stats.getMax()));
+ }
+
+ private static void verifyDoubleStats(Scenario scenario, String name, DoubleStatistics stats) {
+ if (scenario == Scenario.NO_NAN) {
+ assertEquals(0L, stats.getNanCount());
+ assertEquals(-2d, stats.getMin(), 0d);
+ assertEquals(5d, stats.getMax(), 0d);
+ return;
+ }
+
+ if (scenario == Scenario.MIXED_NAN) {
+ assertEquals(4L, stats.getNanCount());
+ if (isIeee(name)) {
+ assertEquals(-2d, stats.getMin(), 0d);
+ assertEquals(3d, stats.getMax(), 0d);
+ }
+ return;
+ }
+
+ if (scenario == Scenario.ALL_NAN) {
+ assertEquals(10L, stats.getNanCount());
+ if (isIeee(name)) {
+ assertEquals(0xffffffffffffffffL, Double.doubleToRawLongBits(stats.getMin()));
+ assertEquals(0x7fffffffffffffffL, Double.doubleToRawLongBits(stats.getMax()));
+ }
+ return;
+ }
+
+ if (scenario == Scenario.ZERO_MIN) {
+ assertEquals(0L, stats.getNanCount());
+ assertEquals(
+ isIeee(name) ? 0x0000000000000000L : 0x8000000000000000L,
+ Double.doubleToRawLongBits(stats.getMin()));
+ assertEquals(5d, stats.getMax(), 0d);
+ return;
+ }
+
+ assertEquals(0L, stats.getNanCount());
+ assertEquals(-5d, stats.getMin(), 0d);
+ assertEquals(
+ isIeee(name) ? 0x8000000000000000L : 0x0000000000000000L, Double.doubleToRawLongBits(stats.getMax()));
+ }
+
+ private static void verifyFloat16Stats(Scenario scenario, String name, BinaryStatistics stats) {
+ if (scenario == Scenario.NO_NAN) {
+ short min = stats.genericGetMin().get2BytesLittleEndian();
+ short max = stats.genericGetMax().get2BytesLittleEndian();
+ assertEquals(0L, stats.getNanCount());
+ assertEquals((short) 0xc000, min);
+ assertEquals((short) 0x4500, max);
+ return;
+ }
+
+ if (scenario == Scenario.MIXED_NAN) {
+ assertEquals(4L, stats.getNanCount());
+ if (isIeee(name)) {
+ short min = stats.genericGetMin().get2BytesLittleEndian();
+ short max = stats.genericGetMax().get2BytesLittleEndian();
+ assertEquals((short) 0xc000, min);
+ assertEquals((short) 0x4200, max);
+ }
+ return;
+ }
+
+ if (scenario == Scenario.ALL_NAN) {
+ assertEquals(10L, stats.getNanCount());
+ if (isIeee(name)) {
+ short min = stats.genericGetMin().get2BytesLittleEndian();
+ short max = stats.genericGetMax().get2BytesLittleEndian();
+ assertEquals((short) 0xffff, min);
+ assertEquals((short) 0x7fff, max);
+ }
+ return;
+ }
+
+ if (scenario == Scenario.ZERO_MIN) {
+ short min = stats.genericGetMin().get2BytesLittleEndian();
+ short max = stats.genericGetMax().get2BytesLittleEndian();
+ assertEquals(0L, stats.getNanCount());
+ assertEquals((short) (isIeee(name) ? 0x0000 : 0x8000), min);
+ assertEquals((short) 0x4500, max);
+ return;
+ }
+
+ assertEquals(0L, stats.getNanCount());
+ if (isIeee(name)) {
+ short min = stats.genericGetMin().get2BytesLittleEndian();
+ short max = stats.genericGetMax().get2BytesLittleEndian();
+ assertEquals((short) 0xc500, min);
+ assertEquals((short) 0x8000, max);
+ } else {
+ short min = stats.genericGetMin().get2BytesLittleEndian();
+ short max = stats.genericGetMax().get2BytesLittleEndian();
+ assertEquals((short) 0xc500, min);
+ assertEquals((short) 0x0000, max);
+ }
+ }
+
+ private static void verifyColumnIndex(ParquetFileReader reader, Scenario scenario, ColumnChunkMetaData column)
+ throws IOException {
+ String name = column.getPath().toDotString();
+ boolean ieee = isIeee(name);
+ boolean hasNaN = scenario == Scenario.MIXED_NAN || scenario == Scenario.ALL_NAN;
+
+ ColumnIndex columnIndex = reader.readColumnIndex(column);
+ if (!ieee && hasNaN) {
+ // TypeDefinedOrder + NaN => page index is intentionally invalidated.
+ assertNull(columnIndex);
+ return;
+ }
+
+ assertNotNull(columnIndex);
+ assertNotNull(columnIndex.getNanCounts());
+ assertEquals(1, columnIndex.getNanCounts().size());
+ assertEquals(
+ expectedNanCount(scenario), (long) columnIndex.getNanCounts().get(0));
+
+ assertEquals(1, columnIndex.getMinValues().size());
+ assertEquals(1, columnIndex.getMaxValues().size());
+ ByteBuffer min = columnIndex.getMinValues().get(0).order(ByteOrder.LITTLE_ENDIAN);
+ ByteBuffer max = columnIndex.getMaxValues().get(0).order(ByteOrder.LITTLE_ENDIAN);
+
+ if (name.contains("float16")) {
+ verifyFloat16ColumnIndexBounds(scenario, ieee, min, max);
+ return;
+ }
+
+ if (name.contains("double")) {
+ verifyDoubleColumnIndexBounds(scenario, ieee, min, max);
+ return;
+ }
+
+ verifyFloatColumnIndexBounds(scenario, ieee, min, max);
+ }
+
+ private static long expectedNanCount(Scenario scenario) {
+ if (scenario == Scenario.MIXED_NAN) {
+ return 4L;
+ }
+ if (scenario == Scenario.ALL_NAN) {
+ return 10L;
+ }
+ return 0L;
+ }
+
+ private static void verifyFloat16ColumnIndexBounds(
+ Scenario scenario, boolean ieee, ByteBuffer min, ByteBuffer max) {
+ short minV = min.getShort(0);
+ short maxV = max.getShort(0);
+ if (scenario == Scenario.ALL_NAN) {
+ assertEquals((short) 0xffff, minV);
+ assertEquals((short) 0x7fff, maxV);
+ } else if (scenario == Scenario.MIXED_NAN || scenario == Scenario.NO_NAN) {
+ assertEquals((short) 0xc000, minV);
+ assertEquals((short) (scenario == Scenario.MIXED_NAN ? 0x4200 : 0x4500), maxV);
+ } else if (scenario == Scenario.ZERO_MIN) {
+ assertEquals((short) (ieee ? 0x0000 : 0x8000), minV);
+ assertEquals((short) 0x4500, maxV);
+ } else {
+ assertEquals((short) 0xc500, minV);
+ assertEquals((short) (ieee ? 0x8000 : 0x0000), maxV);
+ }
+ }
+
+ private static void verifyDoubleColumnIndexBounds(Scenario scenario, boolean ieee, ByteBuffer min, ByteBuffer max) {
+ long minV = min.getLong(0);
+ long maxV = max.getLong(0);
+ if (scenario == Scenario.ALL_NAN) {
+ assertEquals(0xffffffffffffffffL, minV);
+ assertEquals(0x7fffffffffffffffL, maxV);
+ } else if (scenario == Scenario.MIXED_NAN || scenario == Scenario.NO_NAN) {
+ assertEquals(Double.doubleToRawLongBits(-2d), minV);
+ assertEquals(Double.doubleToRawLongBits(scenario == Scenario.MIXED_NAN ? 3d : 5d), maxV);
+ } else if (scenario == Scenario.ZERO_MIN) {
+ assertEquals(Double.doubleToRawLongBits(ieee ? 0d : -0d), minV);
+ assertEquals(Double.doubleToRawLongBits(5d), maxV);
+ } else {
+ assertEquals(Double.doubleToRawLongBits(-5d), minV);
+ assertEquals(Double.doubleToRawLongBits(ieee ? -0d : 0d), maxV);
+ }
+ }
+
+ private static void verifyFloatColumnIndexBounds(Scenario scenario, boolean ieee, ByteBuffer min, ByteBuffer max) {
+ int minV = min.getInt(0);
+ int maxV = max.getInt(0);
+ if (scenario == Scenario.ALL_NAN) {
+ assertEquals(0xffffffff, minV);
+ assertEquals(0x7fffffff, maxV);
+ } else if (scenario == Scenario.MIXED_NAN || scenario == Scenario.NO_NAN) {
+ assertEquals(Float.floatToRawIntBits(-2f), minV);
+ assertEquals(Float.floatToRawIntBits(scenario == Scenario.MIXED_NAN ? 3f : 5f), maxV);
+ } else if (scenario == Scenario.ZERO_MIN) {
+ assertEquals(Float.floatToRawIntBits(ieee ? 0f : -0f), minV);
+ assertEquals(Float.floatToRawIntBits(5f), maxV);
+ } else {
+ assertEquals(Float.floatToRawIntBits(-5f), minV);
+ assertEquals(Float.floatToRawIntBits(ieee ? -0f : 0f), maxV);
+ }
+ }
+
+ private static boolean isIeee(String columnName) {
+ return columnName.endsWith("ieee754");
+ }
+
+ private static void assertFloatValue(float expected, float actual) {
+ assertEquals(Float.floatToRawIntBits(expected), Float.floatToRawIntBits(actual));
+ }
+
+ private static void assertDoubleValue(double expected, double actual) {
+ assertEquals(Double.doubleToRawLongBits(expected), Double.doubleToRawLongBits(actual));
+ }
+
+ private static List<Float> concatFloatValues(Scenario... scenarios) {
+ List<Float> values = new ArrayList<>();
+ for (Scenario scenario : scenarios) {
+ for (float value : FloatingPointNanInteropFileGenerator.floatValues(scenario)) {
+ values.add(value);
+ }
+ }
+ return values;
+ }
+
+ private static List<Double> concatDoubleValues(Scenario... scenarios) {
+ List<Double> values = new ArrayList<>();
+ for (Scenario scenario : scenarios) {
+ for (double value : FloatingPointNanInteropFileGenerator.doubleValues(scenario)) {
+ values.add(value);
+ }
+ }
+ return values;
+ }
+
+ private static List<Binary> concatFloat16Values(Scenario... scenarios) {
+ List<Binary> values = new ArrayList<>();
+ for (Scenario scenario : scenarios) {
+ for (Binary value : FloatingPointNanInteropFileGenerator.float16Values(scenario)) {
+ values.add(value);
+ }
+ }
+ return values;
+ }
+}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestFloat16Statistics.java b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestFloat16Statistics.java
index 38d362d5f..ec445af9c 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestFloat16Statistics.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestFloat16Statistics.java
@@ -132,11 +132,11 @@ public class TestFloat16Statistics {
Binary.fromConstantByteArray(new byte[] {(byte) 0x00, (byte) 0x7e})
}; // NaN
- // Float16Builder: Drop min/max values in case of NaN as the sorting order of values is undefined
+ // Float16Statistics: NaN values are counted but excluded from min/max
private Binary[] valuesWithNaNStatsMinMax = {
- Binary.fromConstantByteArray(new byte[] {(byte) 0x00, (byte) 0x00}), // +0
- Binary.fromConstantByteArray(new byte[] {(byte) 0x00, (byte) 0x00})
- }; // +0
+ Binary.fromConstantByteArray(new byte[] {(byte) 0x00, (byte) 0xc0}), // -2.0
+ Binary.fromConstantByteArray(new byte[] {(byte) 0xff, (byte) 0x7b})
+ }; // 65504.0
@Test
public void testFloat16StatisticsMultipleCases() throws IOException {
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestIeee754TotalOrderE2E.java b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestIeee754TotalOrderE2E.java
new file mode 100644
index 000000000..06376829a
--- /dev/null
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestIeee754TotalOrderE2E.java
@@ -0,0 +1,546 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.statistics;
+
+import static org.apache.parquet.filter2.predicate.FilterApi.binaryColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.doubleColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.eq;
+import static org.apache.parquet.filter2.predicate.FilterApi.floatColumn;
+import static org.apache.parquet.filter2.predicate.FilterApi.notEq;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.parquet.Preconditions;
+import org.apache.parquet.column.statistics.DoubleStatistics;
+import org.apache.parquet.column.statistics.FloatStatistics;
+import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.example.data.Group;
+import org.apache.parquet.example.data.GroupFactory;
+import org.apache.parquet.example.data.simple.SimpleGroupFactory;
+import org.apache.parquet.filter2.compat.FilterCompat;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.example.ExampleParquetWriter;
+import org.apache.parquet.hadoop.example.GroupReadSupport;
+import org.apache.parquet.hadoop.metadata.BlockMetaData;
+import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
+import org.apache.parquet.hadoop.util.HadoopInputFile;
+import org.apache.parquet.internal.column.columnindex.ColumnIndex;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.schema.ColumnOrder;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Types;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class TestIeee754TotalOrderE2E {
+
+ private static final Binary FLOAT16_NAN_A = Binary.fromConstantByteArray(new byte[] {0x01, 0x7e});
+ private static final Binary FLOAT16_NAN_B = Binary.fromConstantByteArray(new byte[] {0x02, 0x7e});
+ private static final Binary FLOAT16_ONE = Binary.fromConstantByteArray(new byte[] {0x00, 0x3c});
+
+ @Rule
+ public TemporaryFolder temp = new TemporaryFolder();
+
+ private static final MessageType FLOAT_SCHEMA = Types.buildMessage()
+ .required(PrimitiveTypeName.FLOAT)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("float_col")
+ .named("msg");
+
+ private static final MessageType DOUBLE_SCHEMA = Types.buildMessage()
+ .required(PrimitiveTypeName.DOUBLE)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("double_col")
+ .named("msg");
+
+ private static final MessageType FLOAT_DOUBLE_SCHEMA = Types.buildMessage()
+ .required(PrimitiveTypeName.FLOAT)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("float_col")
+ .required(PrimitiveTypeName.DOUBLE)
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("double_col")
+ .named("msg");
+
+ private static final MessageType FLOAT16_SCHEMA = Types.buildMessage()
+ .required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(LogicalTypeAnnotation.Float16LogicalTypeAnnotation.BYTES)
+ .as(LogicalTypeAnnotation.float16Type())
+ .columnOrder(ColumnOrder.ieee754TotalOrder())
+ .named("float16_col")
+ .named("msg");
+
+ private static final MessageType TYPE_DEFINED_FLOAT_SCHEMA = Types.buildMessage()
+ .required(PrimitiveTypeName.FLOAT)
+ .columnOrder(ColumnOrder.typeDefined())
+ .named("float_col")
+ .named("msg");
+
+ private static final MessageType TYPE_DEFINED_DOUBLE_SCHEMA = Types.buildMessage()
+ .required(PrimitiveTypeName.DOUBLE)
+ .columnOrder(ColumnOrder.typeDefined())
+ .named("double_col")
+ .named("msg");
+
+ private static final MessageType TYPE_DEFINED_FLOAT16_SCHEMA = Types.buildMessage()
+ .required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+ .length(LogicalTypeAnnotation.Float16LogicalTypeAnnotation.BYTES)
+ .as(LogicalTypeAnnotation.float16Type())
+ .columnOrder(ColumnOrder.typeDefined())
+ .named("float16_col")
+ .named("msg");
+
+ private Path newTempPath() throws IOException {
+ File file = temp.newFile();
+ Preconditions.checkArgument(file.delete(), "Could not remove temp file");
+ return new Path(file.getAbsolutePath());
+ }
+
+ private Path writeFloatFile(float... values) throws IOException {
+ return writeFloatFile(FLOAT_SCHEMA, values);
+ }
+
+ private Path writeFloatFile(MessageType schema, float... values) throws IOException {
+ Path path = newTempPath();
+ try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(path)
+ .withType(schema)
+ .withDictionaryEncoding(false)
+ .build()) {
+ GroupFactory factory = new SimpleGroupFactory(schema);
+ for (float v : values) {
+ writer.write(factory.newGroup().append("float_col", v));
+ }
+ }
+ return path;
+ }
+
+ private Path writeDoubleFile(double... values) throws IOException {
+ return writeDoubleFile(DOUBLE_SCHEMA, values);
+ }
+
+ private Path writeDoubleFile(MessageType schema, double... values) throws IOException {
+ Path path = newTempPath();
+ try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(path)
+ .withType(schema)
+ .withDictionaryEncoding(false)
+ .build()) {
+ GroupFactory factory = new SimpleGroupFactory(schema);
+ for (double v : values) {
+ writer.write(factory.newGroup().append("double_col", v));
+ }
+ }
+ return path;
+ }
+
+ private Path writeFloat16File(MessageType schema, Binary... values) throws IOException {
+ Path path = newTempPath();
+ try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(path)
+ .withType(schema)
+ .withDictionaryEncoding(false)
+ .build()) {
+ GroupFactory factory = new SimpleGroupFactory(schema);
+ for (Binary v : values) {
+ writer.write(factory.newGroup().append("float16_col", v));
+ }
+ }
+ return path;
+ }
+
+ private Path writeFloatDoubleFile(float[] floatValues, double[] doubleValues) throws IOException {
+ Preconditions.checkArgument(floatValues.length == doubleValues.length, "Arrays must have same length");
+ Path path = newTempPath();
+ try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(path)
+ .withType(FLOAT_DOUBLE_SCHEMA)
+ .withDictionaryEncoding(false)
+ .build()) {
+ GroupFactory factory = new SimpleGroupFactory(FLOAT_DOUBLE_SCHEMA);
+ for (int i = 0; i < floatValues.length; i++) {
+ writer.write(
+ factory.newGroup().append("float_col", floatValues[i]).append("double_col", doubleValues[i]));
+ }
+ }
+ return path;
+ }
+
+ private List<Float> readFloatValues(Path path) throws IOException {
+ return readFloatValues(path, null);
+ }
+
+ private List<Float> readFloatValues(Path path, FilterCompat.Filter filter) throws IOException {
+ List<Float> result = new ArrayList<>();
+ ParquetReader.Builder<Group> builder = ParquetReader.builder(new GroupReadSupport(), path);
+ if (filter != null) {
+ builder.withFilter(filter);
+ }
+ try (ParquetReader<Group> reader = builder.build()) {
+ Group group;
+ while ((group = reader.read()) != null) {
+ result.add(group.getFloat("float_col", 0));
+ }
+ }
+ return result;
+ }
+
+ private List<Float> readFloatValuesWithRecordFilter(Path path, FilterCompat.Filter filter) throws IOException {
+ List<Float> result = new ArrayList<>();
+ ParquetReader.Builder<Group> builder = ParquetReader.builder(new GroupReadSupport(), path)
+ .useBloomFilter(false)
+ .useDictionaryFilter(false)
+ .useStatsFilter(false)
+ .useColumnIndexFilter(false)
+ .withFilter(filter);
+ try (ParquetReader<Group> reader = builder.build()) {
+ Group group;
+ while ((group = reader.read()) != null) {
+ result.add(group.getFloat("float_col", 0));
+ }
+ }
+ return result;
+ }
+
+ private List<Double> readDoubleValues(Path path) throws IOException {
+ return readDoubleValues(path, null);
+ }
+
+ private List<Double> readDoubleValues(Path path, FilterCompat.Filter filter) throws IOException {
+ List<Double> result = new ArrayList<>();
+ ParquetReader.Builder<Group> builder = ParquetReader.builder(new GroupReadSupport(), path);
+ if (filter != null) {
+ builder.withFilter(filter);
+ }
+ try (ParquetReader<Group> reader = builder.build()) {
+ Group group;
+ while ((group = reader.read()) != null) {
+ result.add(group.getDouble("double_col", 0));
+ }
+ }
+ return result;
+ }
+
+ private List<Double> readDoubleValuesWithRecordFilter(Path path, FilterCompat.Filter filter) throws IOException {
+ List<Double> result = new ArrayList<>();
+ ParquetReader.Builder<Group> builder = ParquetReader.builder(new GroupReadSupport(), path)
+ .useBloomFilter(false)
+ .useDictionaryFilter(false)
+ .useStatsFilter(false)
+ .useColumnIndexFilter(false)
+ .withFilter(filter);
+ try (ParquetReader<Group> reader = builder.build()) {
+ Group group;
+ while ((group = reader.read()) != null) {
+ result.add(group.getDouble("double_col", 0));
+ }
+ }
+ return result;
+ }
+
+ private List<Binary> readFloat16ValuesWithRecordFilter(Path path, FilterCompat.Filter filter) throws IOException {
+ List<Binary> result = new ArrayList<>();
+ ParquetReader.Builder<Group> builder = ParquetReader.builder(new GroupReadSupport(), path)
+ .useBloomFilter(false)
+ .useDictionaryFilter(false)
+ .useStatsFilter(false)
+ .useColumnIndexFilter(false)
+ .withFilter(filter);
+ try (ParquetReader<Group> reader = builder.build()) {
+ Group group;
+ while ((group = reader.read()) != null) {
+ result.add(group.getBinary("float16_col", 0));
+ }
+ }
+ return result;
+ }
+
+ private static void assertFloatBits(List<Float> values, int... expectedBits) {
+ assertEquals(expectedBits.length, values.size());
+ for (int i = 0; i < expectedBits.length; i++) {
+ assertEquals(expectedBits[i], Float.floatToRawIntBits(values.get(i)));
+ }
+ }
+
+ private static void assertDoubleBits(List<Double> values, long... expectedBits) {
+ assertEquals(expectedBits.length, values.size());
+ for (int i = 0; i < expectedBits.length; i++) {
+ assertEquals(expectedBits[i], Double.doubleToRawLongBits(values.get(i)));
+ }
+ }
+
+ private static void assertFloat16Bits(List<Binary> values, int... expectedBits) {
+ assertEquals(expectedBits.length, values.size());
+ for (int i = 0; i < expectedBits.length; i++) {
+ assertEquals(expectedBits[i], values.get(i).get2BytesLittleEndian());
+ }
+ }
+
+ @Test
+ public void testFloatStatisticsWithNaN() throws IOException {
+ Path path = writeFloatFile(1.0f, Float.NaN, 3.0f, Float.NaN, 5.0f);
+
+ try (ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(path, new Configuration()))) {
+ BlockMetaData block = reader.getFooter().getBlocks().get(0);
+ ColumnChunkMetaData column = block.getColumns().get(0);
+
+ // Verify column order is IEEE 754
+ assertEquals(
+ ColumnOrder.ieee754TotalOrder(), column.getPrimitiveType().columnOrder());
+
+ // Verify statistics
+ Statistics<?> stats = column.getStatistics();
+ assertNotNull(stats);
+ assertTrue("nan_count should be set", stats.isNanCountSet());
+ assertEquals("nan_count should be 2", 2, stats.getNanCount());
+
+ // Min/max should exclude NaN
+ FloatStatistics floatStats = (FloatStatistics) stats;
+ assertEquals(1.0f, floatStats.getMin(), 0.0f);
+ assertEquals(5.0f, floatStats.getMax(), 0.0f);
+
+ // Verify column index
+ ColumnIndex columnIndex = reader.readColumnIndex(column);
+ assertNotNull("ColumnIndex should not be null for IEEE 754 with NaN", columnIndex);
+ List<Long> nanCounts = columnIndex.getNanCounts();
+ assertNotNull("nan_counts should be set in column index", nanCounts);
+ // All values in one page, so one entry with nan_count = 2
+ assertEquals(1, nanCounts.size());
+ assertEquals(Long.valueOf(2), nanCounts.get(0));
+
+ // Verify min/max in column index exclude NaN
+ List<ByteBuffer> minValues = columnIndex.getMinValues();
+ List<ByteBuffer> maxValues = columnIndex.getMaxValues();
+ assertEquals(1, minValues.size());
+ float ciMin = minValues.get(0).order(ByteOrder.LITTLE_ENDIAN).getFloat(0);
+ float ciMax = maxValues.get(0).order(ByteOrder.LITTLE_ENDIAN).getFloat(0);
+ assertEquals(1.0f, ciMin, 0.0f);
+ assertEquals(5.0f, ciMax, 0.0f);
+ }
+ }
+
+ @Test
+ public void testDoubleStatisticsWithNaN() throws IOException {
+ // Write: -10.0, NaN, 20.0, NaN, NaN
+ Path path = writeDoubleFile(-10.0, Double.NaN, 20.0, Double.NaN, Double.NaN);
+
+ try (ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(path, new Configuration()))) {
+ BlockMetaData block = reader.getFooter().getBlocks().get(0);
+ ColumnChunkMetaData column = block.getColumns().get(0);
+
+ assertEquals(
+ ColumnOrder.ieee754TotalOrder(), column.getPrimitiveType().columnOrder());
+
+ Statistics<?> stats = column.getStatistics();
+ assertTrue(stats.isNanCountSet());
+ assertEquals(3, stats.getNanCount());
+
+ DoubleStatistics doubleStats = (DoubleStatistics) stats;
+ assertEquals(-10.0, doubleStats.getMin(), 0.0);
+ assertEquals(20.0, doubleStats.getMax(), 0.0);
+
+ ColumnIndex columnIndex = reader.readColumnIndex(column);
+ assertNotNull(columnIndex);
+ List<Long> nanCounts = columnIndex.getNanCounts();
+ assertNotNull(nanCounts);
+ assertEquals(1, nanCounts.size());
+ assertEquals(Long.valueOf(3), nanCounts.get(0));
+ }
+ }
+
+ @Test
+ public void testFloatStatisticsNoNaN() throws IOException {
+ Path path = writeFloatFile(1.0f, 2.0f, 3.0f);
+
+ try (ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(path, new Configuration()))) {
+ ColumnChunkMetaData column =
+ reader.getFooter().getBlocks().get(0).getColumns().get(0);
+ Statistics<?> stats = column.getStatistics();
+
+ assertTrue(stats.isNanCountSet());
+ assertEquals(0, stats.getNanCount());
+
+ FloatStatistics floatStats = (FloatStatistics) stats;
+ assertEquals(1.0f, floatStats.getMin(), 0.0f);
+ assertEquals(3.0f, floatStats.getMax(), 0.0f);
+ }
+ }
+
+ @Test
+ public void testFloatStatisticsAllNaN() throws IOException {
+ Path path = writeFloatFile(Float.NaN, Float.NaN, Float.NaN);
+
+ try (ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(path, new Configuration()))) {
+ ColumnChunkMetaData column =
+ reader.getFooter().getBlocks().get(0).getColumns().get(0);
+ Statistics<?> stats = column.getStatistics();
+
+ assertTrue(stats.isNanCountSet());
+ assertEquals(3, stats.getNanCount());
+
+ assertTrue("All-NaN column should have min/max values", stats.hasNonNullValue());
+ FloatStatistics floatStats = (FloatStatistics) stats;
+ assertTrue("All-NaN min should be NaN", Float.isNaN(floatStats.getMin()));
+ assertTrue("All-NaN max should be NaN", Float.isNaN(floatStats.getMax()));
+ }
+ }
+
+ @Test
+ public void testRecordLevelFloatFiltersDistinguishNaNRawBitsWithIeee754TotalOrder() throws IOException {
+ float nanA = Float.intBitsToFloat(0x7fc00001);
+ float nanB = Float.intBitsToFloat(0x7fc00002);
+ Path path = writeFloatFile(nanA, nanB, 1.0f);
+
+ assertFloatBits(
+ readFloatValuesWithRecordFilter(path, FilterCompat.get(eq(floatColumn("float_col"), nanA))),
+ 0x7fc00001);
+ assertFloatBits(
+ readFloatValuesWithRecordFilter(path, FilterCompat.get(notEq(floatColumn("float_col"), nanA))),
+ 0x7fc00002,
+ Float.floatToRawIntBits(1.0f));
+ }
+
+ @Test
+ public void testRecordLevelDoubleFiltersDistinguishNaNRawBitsWithIeee754TotalOrder() throws IOException {
+ double nanA = Double.longBitsToDouble(0x7ff8000000000001L);
+ double nanB = Double.longBitsToDouble(0x7ff8000000000002L);
+ Path path = writeDoubleFile(nanA, nanB, 1.0);
+
+ assertDoubleBits(
+ readDoubleValuesWithRecordFilter(path, FilterCompat.get(eq(doubleColumn("double_col"), nanA))),
+ 0x7ff8000000000001L);
+ assertDoubleBits(
+ readDoubleValuesWithRecordFilter(path, FilterCompat.get(notEq(doubleColumn("double_col"), nanA))),
+ 0x7ff8000000000002L,
+ Double.doubleToRawLongBits(1.0));
+ }
+
+ @Test
+ public void testRecordLevelFloat16FiltersDistinguishNaNRawBitsWithIeee754TotalOrder() throws IOException {
+ Path path = writeFloat16File(FLOAT16_SCHEMA, FLOAT16_NAN_A, FLOAT16_NAN_B, FLOAT16_ONE);
+
+ assertFloat16Bits(
+ readFloat16ValuesWithRecordFilter(
+ path, FilterCompat.get(eq(binaryColumn("float16_col"), FLOAT16_NAN_A))),
+ 0x7e01);
+ assertFloat16Bits(
+ readFloat16ValuesWithRecordFilter(
+ path, FilterCompat.get(notEq(binaryColumn("float16_col"), FLOAT16_NAN_A))),
+ 0x7e02,
+ 0x3c00);
+ }
+
+ @Test
+ public void testRecordLevelFloatFiltersTreatNaNsEqualWithTypeDefinedOrder() throws IOException {
+ float nanA = Float.intBitsToFloat(0x7fc00001);
+ float nanB = Float.intBitsToFloat(0x7fc00002);
+ Path path = writeFloatFile(TYPE_DEFINED_FLOAT_SCHEMA, nanA, nanB, 1.0f);
+
+ assertFloatBits(
+ readFloatValuesWithRecordFilter(path, FilterCompat.get(eq(floatColumn("float_col"), nanA))),
+ 0x7fc00001,
+ 0x7fc00002);
+ assertFloatBits(
+ readFloatValuesWithRecordFilter(path, FilterCompat.get(notEq(floatColumn("float_col"), nanA))),
+ Float.floatToRawIntBits(1.0f));
+ }
+
+ @Test
+ public void testRecordLevelDoubleFiltersTreatNaNsEqualWithTypeDefinedOrder() throws IOException {
+ double nanA = Double.longBitsToDouble(0x7ff8000000000001L);
+ double nanB = Double.longBitsToDouble(0x7ff8000000000002L);
+ Path path = writeDoubleFile(TYPE_DEFINED_DOUBLE_SCHEMA, nanA, nanB, 1.0);
+
+ assertDoubleBits(
+ readDoubleValuesWithRecordFilter(path, FilterCompat.get(eq(doubleColumn("double_col"), nanA))),
+ 0x7ff8000000000001L,
+ 0x7ff8000000000002L);
+ assertDoubleBits(
+ readDoubleValuesWithRecordFilter(path, FilterCompat.get(notEq(doubleColumn("double_col"), nanA))),
+ Double.doubleToRawLongBits(1.0));
+ }
+
+ @Test
+ public void testRecordLevelFloat16FiltersTreatNaNsEqualWithTypeDefinedOrder() throws IOException {
+ Path path = writeFloat16File(TYPE_DEFINED_FLOAT16_SCHEMA, FLOAT16_NAN_A, FLOAT16_NAN_B, FLOAT16_ONE);
+
+ assertFloat16Bits(
+ readFloat16ValuesWithRecordFilter(
+ path, FilterCompat.get(eq(binaryColumn("float16_col"), FLOAT16_NAN_A))),
+ 0x7e01,
+ 0x7e02);
+ assertFloat16Bits(
+ readFloat16ValuesWithRecordFilter(
+ path, FilterCompat.get(notEq(binaryColumn("float16_col"), FLOAT16_NAN_A))),
+ 0x3c00);
+ }
+
+ @Test
+ public void testFloatDoubleColumnsWithNaN() throws IOException {
+ float[] floatValues = {1.0f, Float.NaN, 3.0f};
+ double[] doubleValues = {-5.0, Double.NaN, 10.0};
+ Path path = writeFloatDoubleFile(floatValues, doubleValues);
+
+ try (ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(path, new Configuration()))) {
+ BlockMetaData block = reader.getFooter().getBlocks().get(0);
+
+ // Verify float column
+ ColumnChunkMetaData floatCol = block.getColumns().get(0);
+ assertEquals(
+ ColumnOrder.ieee754TotalOrder(), floatCol.getPrimitiveType().columnOrder());
+ Statistics<?> floatStats = floatCol.getStatistics();
+ assertTrue(floatStats.isNanCountSet());
+ assertEquals(1, floatStats.getNanCount());
+ assertEquals(1.0f, ((FloatStatistics) floatStats).getMin(), 0.0f);
+ assertEquals(3.0f, ((FloatStatistics) floatStats).getMax(), 0.0f);
+
+ // Verify double column
+ ColumnChunkMetaData doubleCol = block.getColumns().get(1);
+ assertEquals(
+ ColumnOrder.ieee754TotalOrder(),
+ doubleCol.getPrimitiveType().columnOrder());
+ Statistics<?> doubleStats = doubleCol.getStatistics();
+ assertTrue(doubleStats.isNanCountSet());
+ assertEquals(1, doubleStats.getNanCount());
+ assertEquals(-5.0, ((DoubleStatistics) doubleStats).getMin(), 0.0);
+ assertEquals(10.0, ((DoubleStatistics) doubleStats).getMax(), 0.0);
+
+ // Verify column indexes for both
+ ColumnIndex floatCI = reader.readColumnIndex(floatCol);
+ assertNotNull(floatCI);
+ assertNotNull(floatCI.getNanCounts());
+ assertEquals(Long.valueOf(1), floatCI.getNanCounts().get(0));
+
+ ColumnIndex doubleCI = reader.readColumnIndex(doubleCol);
+ assertNotNull(doubleCI);
+ assertNotNull(doubleCI.getNanCounts());
+ assertEquals(Long.valueOf(1), doubleCI.getNanCounts().get(0));
+ }
+ }
+}