You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2026/07/24 00:11:17 UTC
(iceberg) branch main updated: API: Extract superclass from InclusiveMetricsEvaluator (#17201)
This is an automated email from the ASF dual-hosted git repository.
rdblue pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new c29264684b API: Extract superclass from InclusiveMetricsEvaluator (#17201)
c29264684b is described below
commit c29264684b3232135c526c2bafb3be9b5357bce4
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Thu Jul 23 17:11:10 2026 -0700
API: Extract superclass from InclusiveMetricsEvaluator (#17201)
---
.../iceberg/expressions/InclusiveEvalVisitor.java | 532 +++++++++++++++++++++
.../expressions/InclusiveMetricsEvaluator.java | 503 +------------------
.../expressions/TestInclusiveMetricsEvaluator.java | 46 +-
3 files changed, 595 insertions(+), 486 deletions(-)
diff --git a/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java b/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java
new file mode 100644
index 0000000000..02ee2c7ffc
--- /dev/null
+++ b/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java
@@ -0,0 +1,532 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.expressions;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.util.NaNUtil;
+
+abstract class InclusiveEvalVisitor extends ExpressionVisitors.BoundVisitor<Boolean> {
+ private static final int IN_PREDICATE_LIMIT = 200;
+
+ protected static final boolean ROWS_MIGHT_MATCH = true;
+ protected static final boolean ROWS_CANNOT_MATCH = false;
+
+ /** Return true if null count is non-zero or is unknown, false if null count is 0. */
+ protected abstract boolean mayContainNull(int id);
+
+ /** Return true if null count is known and equal to value count, false otherwise. */
+ protected abstract boolean containsNullsOnly(int id);
+
+ /** Return true if NaN count is non-zero or is unknown, false if NaN count is 0. */
+ protected abstract boolean mayContainNaN(int id);
+
+ /** Return true if NaN count is known and equal to value count, false otherwise. */
+ protected abstract boolean containsNaNsOnly(int id);
+
+ /** Return the lower bound if it is known, or null otherwise. */
+ protected <T> T lowerBound(BoundReference<T> ref) {
+ return null;
+ }
+
+ /** Return the upper bound if it is known, or null otherwise. */
+ protected <T> T upperBound(BoundReference<T> ref) {
+ return null;
+ }
+
+ /** Return a variant field's lower bound if it is known, or null otherwise. */
+ protected <T> T extractLowerBound(BoundExtract<T> bound) {
+ return null;
+ }
+
+ /** Return a variant field's upper bound if it is known, or null otherwise. */
+ protected <T> T extractUpperBound(BoundExtract<T> bound) {
+ return null;
+ }
+
+ @Override
+ public Boolean alwaysTrue() {
+ return ROWS_MIGHT_MATCH; // all rows match
+ }
+
+ @Override
+ public Boolean alwaysFalse() {
+ return ROWS_CANNOT_MATCH; // all rows fail
+ }
+
+ @Override
+ public Boolean not(Boolean result) {
+ return !result;
+ }
+
+ @Override
+ public Boolean and(Boolean leftResult, Boolean rightResult) {
+ return leftResult && rightResult;
+ }
+
+ @Override
+ public Boolean or(Boolean leftResult, Boolean rightResult) {
+ return leftResult || rightResult;
+ }
+
+ @Override
+ public <T> Boolean isNull(Bound<T> term) {
+ // no need to check whether the field is required because binding evaluates that case
+ // if the column has no null values, the expression cannot match
+ if (isNonNullPreserving(term)) {
+ // number of non-nulls is the same as for the ref
+ int id = term.ref().fieldId();
+ if (!mayContainNull(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean notNull(Bound<T> term) {
+ // no need to check whether the field is required because binding evaluates that case
+ // if the column has no non-null values, the expression cannot match
+
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean isNaN(Bound<T> term) {
+ // when there's no nanCounts information, but we already know the column only contains null,
+ // it's guaranteed that there's no NaN value
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ if (!(term instanceof BoundReference)) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ if (!mayContainNaN(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean notNaN(Bound<T> term) {
+ if (!(term instanceof BoundReference)) {
+ // identity transforms are already removed by this time
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int id = term.ref().fieldId();
+
+ if (containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean lt(Bound<T> term, Literal<T> lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T lower = evalLowerBound(term);
+ if (null == lower || NaNUtil.isNaN(lower)) {
+ // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more.
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // this also works for transforms that are order preserving:
+ // if a transform f is order preserving, a < b means that f(a) <= f(b).
+ // because lower <= a for all values of a in the file, f(lower) <= f(a).
+ // when f(lower) >= X then f(a) >= f(lower) >= X, so there is no a such that f(a) < X
+ // f(lower) >= X means rows cannot match
+ int cmp = lit.comparator().compare(lower, lit.value());
+ if (cmp >= 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean ltEq(Bound<T> term, Literal<T> lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T lower = evalLowerBound(term);
+ if (null == lower || NaNUtil.isNaN(lower)) {
+ // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more.
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // this also works for transforms that are order preserving:
+ // if a transform f is order preserving, a < b means that f(a) <= f(b).
+ // because lower <= a for all values of a in the file, f(lower) <= f(a).
+ // when f(lower) > X then f(a) >= f(lower) > X, so there is no a such that f(a) <= X
+ // f(lower) > X means rows cannot match
+ int cmp = lit.comparator().compare(lower, lit.value());
+ if (cmp > 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean gt(Bound<T> term, Literal<T> lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T upper = evalUpperBound(term);
+ if (null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int cmp = lit.comparator().compare(upper, lit.value());
+ if (cmp <= 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean gtEq(Bound<T> term, Literal<T> lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T upper = evalUpperBound(term);
+ if (null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int cmp = lit.comparator().compare(upper, lit.value());
+ if (cmp < 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean eq(Bound<T> term, Literal<T> lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T lower = evalLowerBound(term);
+ if (lower != null && !NaNUtil.isNaN(lower)) {
+ int cmp = lit.comparator().compare(lower, lit.value());
+ if (cmp > 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+ }
+
+ T upper = evalUpperBound(term);
+ if (null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int cmp = lit.comparator().compare(upper, lit.value());
+ if (cmp < 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean notEq(Bound<T> term, Literal<T> lit) {
+ // because the bounds are not necessarily a min or max value, this cannot be answered using
+ // them. notEq(col, X) with (X, Y) doesn't guarantee that X is a value in col.
+ // However, when min == max and the file has no nulls or NaN values, we can safely prune
+ // if that value equals the literal.
+ T value = uniqueValue(term);
+
+ if (value != null && lit.comparator().compare(value, lit.value()) == 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean in(Bound<T> term, Set<T> literalSet) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ Collection<T> literals = literalSet;
+
+ if (literals.size() > IN_PREDICATE_LIMIT) {
+ // skip evaluating the predicate if the number of values is too big
+ return ROWS_MIGHT_MATCH;
+ }
+
+ T lower = evalLowerBound(term);
+ if (null == lower || NaNUtil.isNaN(lower)) {
+ // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more.
+ return ROWS_MIGHT_MATCH;
+ }
+
+ Comparator<T> comparator = ((BoundTerm<T>) term).comparator();
+
+ literals =
+ literals.stream()
+ .filter(v -> comparator.compare(lower, v) <= 0)
+ .collect(Collectors.toList());
+ // if all values are less than lower bound, rows cannot match
+ if (literals.isEmpty()) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T upper = evalUpperBound(term);
+ if (null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ literals =
+ literals.stream()
+ .filter(v -> comparator.compare(upper, v) >= 0)
+ .collect(Collectors.toList());
+ // if remaining values are greater than upper bound, rows cannot match
+ if (literals.isEmpty()) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean notIn(Bound<T> term, Set<T> literalSet) {
+ // because the bounds are not necessarily a min or max value, this cannot be answered using
+ // them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col.
+ // However, when min == max and the file has no nulls or NaN values, we can safely prune
+ // if that value is in the exclusion set.
+ T value = uniqueValue(term);
+
+ if (value != null && literalSet.contains(value)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean startsWith(Bound<T> term, Literal<T> lit) {
+ if (term instanceof BoundTransform && !((BoundTransform<?, ?>) term).transform().isIdentity()) {
+ // truncate must be rewritten in binding. the result is either always or never compatible
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ String prefix = (String) lit.value();
+
+ Comparator<CharSequence> comparator = Comparators.charSequences();
+
+ CharSequence lower = (CharSequence) evalLowerBound(term);
+ if (null == lower) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // truncate lower bound so that its length in bytes is not greater than the length of prefix
+ int length = Math.min(prefix.length(), lower.length());
+ int cmp = comparator.compare(lower.subSequence(0, length), prefix);
+ if (cmp > 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ CharSequence upper = (CharSequence) evalUpperBound(term);
+ if (null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // truncate upper bound so that its length in bytes is not greater than the length of prefix
+ length = Math.min(prefix.length(), upper.length());
+ cmp = comparator.compare(upper.subSequence(0, length), prefix);
+ if (cmp < 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public <T> Boolean notStartsWith(Bound<T> term, Literal<T> lit) {
+ // the only transforms that produce strings are truncate and identity, which work with this
+ int id = term.ref().fieldId();
+ if (mayContainNull(id)) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ String prefix = (String) lit.value();
+
+ Comparator<CharSequence> comparator = Comparators.charSequences();
+
+ // notStartsWith will match unless all values must start with the prefix. This happens when
+ // the lower and upper bounds both start with the prefix.
+ CharSequence lower = (CharSequence) evalLowerBound(term);
+ CharSequence upper = (CharSequence) evalUpperBound(term);
+ if (null == lower || null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // if lower is shorter than the prefix then lower doesn't start with the prefix
+ if (lower.length() < prefix.length()) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int cmp = comparator.compare(lower.subSequence(0, prefix.length()), prefix);
+ if (cmp == 0) {
+ // if upper is shorter than the prefix then upper can't start with the prefix
+ if (upper.length() < prefix.length()) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ cmp = comparator.compare(upper.subSequence(0, prefix.length()), prefix);
+ if (cmp == 0) {
+ // both bounds match the prefix, so all rows must match the prefix and therefore do not
+ // satisfy the predicate
+ return ROWS_CANNOT_MATCH;
+ }
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ /**
+ * Returns the column's single value if all rows contain the same value. Defined as a column with
+ * no nulls, no NaNs, and lower bound equals upper bound. Returns null otherwise.
+ */
+ private <T> T uniqueValue(Bound<T> term) {
+ int id = term.ref().fieldId();
+ if (mayContainNull(id)) {
+ return null;
+ }
+
+ T lower = evalLowerBound(term);
+ T upper = evalUpperBound(term);
+
+ if (lower == null || upper == null || NaNUtil.isNaN(lower) || NaNUtil.isNaN(upper)) {
+ return null;
+ }
+
+ if ((lower instanceof Float || lower instanceof Double) && mayContainNaN(id)) {
+ return null;
+ }
+
+ if (!lower.equals(upper)) {
+ return null;
+ }
+
+ return lower;
+ }
+
+ private <T> T evalLowerBound(Bound<T> term) {
+ if (term instanceof BoundReference) {
+ return lowerBound((BoundReference<T>) term);
+ } else if (term instanceof BoundTransform) {
+ return transformLowerBound((BoundTransform<?, T>) term);
+ } else if (term instanceof BoundExtract) {
+ return extractLowerBound((BoundExtract<T>) term);
+ } else {
+ return null;
+ }
+ }
+
+ private <T> T evalUpperBound(Bound<T> term) {
+ if (term instanceof BoundReference) {
+ return upperBound((BoundReference<T>) term);
+ } else if (term instanceof BoundTransform) {
+ return transformUpperBound((BoundTransform<?, T>) term);
+ } else if (term instanceof BoundExtract) {
+ return extractUpperBound((BoundExtract<T>) term);
+ } else {
+ return null;
+ }
+ }
+
+ private <S, T> T transformLowerBound(BoundTransform<S, T> boundTransform) {
+ Transform<S, T> transform = boundTransform.transform();
+ if (transform.preservesOrder()) {
+ S lower = lowerBound(boundTransform.ref());
+ return boundTransform.transform().bind(boundTransform.ref().type()).apply(lower);
+ }
+
+ return null;
+ }
+
+ private <S, T> T transformUpperBound(BoundTransform<S, T> boundTransform) {
+ Transform<S, T> transform = boundTransform.transform();
+ if (transform.preservesOrder()) {
+ S upper = upperBound(boundTransform.ref());
+ return boundTransform.transform().bind(boundTransform.ref().type()).apply(upper);
+ }
+
+ return null;
+ }
+
+ /** Returns true if the expression term produces a non-null value for non-null input. */
+ private boolean isNonNullPreserving(Bound<?> term) {
+ if (term instanceof BoundReference) {
+ return true;
+ } else if (term instanceof BoundTransform<?, ?>) {
+ // if the transform preserves order, then non-null values are mapped to non-null
+ return ((BoundTransform<?, ?>) term).transform().preservesOrder();
+ }
+
+ // a non-null variant does not necessarily contain a specific field
+ // and unknown bound terms are not non-null preserving
+ return false;
+ }
+}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java b/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java
index 81cbbe7855..01295e21e3 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java
@@ -21,19 +21,12 @@ package org.apache.iceberg.expressions;
import static org.apache.iceberg.expressions.Expressions.rewriteNot;
import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Comparator;
import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.Schema;
-import org.apache.iceberg.transforms.Transform;
-import org.apache.iceberg.types.Comparators;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Types.StructType;
-import org.apache.iceberg.util.NaNUtil;
import org.apache.iceberg.variants.Variant;
import org.apache.iceberg.variants.VariantObject;
@@ -53,8 +46,6 @@ import org.apache.iceberg.variants.VariantObject;
* checks for NaN is necessary in order to not skip files that may contain matching data.
*/
public class InclusiveMetricsEvaluator {
- private static final int IN_PREDICATE_LIMIT = 200;
-
private final Expression expr;
public InclusiveMetricsEvaluator(Schema schema, Expression unbound) {
@@ -77,10 +68,7 @@ public class InclusiveMetricsEvaluator {
return new MetricsEvalVisitor().eval(file);
}
- private static final boolean ROWS_MIGHT_MATCH = true;
- private static final boolean ROWS_CANNOT_MATCH = false;
-
- private class MetricsEvalVisitor extends ExpressionVisitors.BoundVisitor<Boolean> {
+ private class MetricsEvalVisitor extends InclusiveEvalVisitor {
private Map<Integer, Long> valueCounts = null;
private Map<Integer, Long> nullCounts = null;
private Map<Integer, Long> nanCounts = null;
@@ -109,389 +97,12 @@ public class InclusiveMetricsEvaluator {
}
@Override
- public Boolean alwaysTrue() {
- return ROWS_MIGHT_MATCH; // all rows match
- }
-
- @Override
- public Boolean alwaysFalse() {
- return ROWS_CANNOT_MATCH; // all rows fail
- }
-
- @Override
- public Boolean not(Boolean result) {
- return !result;
- }
-
- @Override
- public Boolean and(Boolean leftResult, Boolean rightResult) {
- return leftResult && rightResult;
- }
-
- @Override
- public Boolean or(Boolean leftResult, Boolean rightResult) {
- return leftResult || rightResult;
- }
-
- @Override
- public <T> Boolean isNull(Bound<T> term) {
- // no need to check whether the field is required because binding evaluates that case
- // if the column has no null values, the expression cannot match
- if (isNonNullPreserving(term)) {
- // number of non-nulls is the same as for the ref
- int id = term.ref().fieldId();
- if (!mayContainNull(id)) {
- return ROWS_CANNOT_MATCH;
- }
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean notNull(Bound<T> term) {
- // no need to check whether the field is required because binding evaluates that case
- // if the column has no non-null values, the expression cannot match
-
- // all terms are null preserving. see #isNullPreserving(Bound)
- int id = term.ref().fieldId();
- if (containsNullsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean isNaN(Bound<T> term) {
- // when there's no nanCounts information, but we already know the column only contains null,
- // it's guaranteed that there's no NaN value
- int id = term.ref().fieldId();
- if (containsNullsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- if (!(term instanceof BoundReference)) {
- return ROWS_MIGHT_MATCH;
- }
-
- if (nanCounts != null && nanCounts.containsKey(id) && nanCounts.get(id) == 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean notNaN(Bound<T> term) {
- if (!(term instanceof BoundReference)) {
- // identity transforms are already removed by this time
- return ROWS_MIGHT_MATCH;
- }
-
- int id = term.ref().fieldId();
-
- if (containsNaNsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean lt(Bound<T> term, Literal<T> lit) {
- // all terms are null preserving. see #isNullPreserving(Bound)
- int id = term.ref().fieldId();
- if (containsNullsOnly(id) || containsNaNsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- T lower = lowerBound(term);
- if (null == lower || NaNUtil.isNaN(lower)) {
- // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more.
- return ROWS_MIGHT_MATCH;
- }
-
- // this also works for transforms that are order preserving:
- // if a transform f is order preserving, a < b means that f(a) <= f(b).
- // because lower <= a for all values of a in the file, f(lower) <= f(a).
- // when f(lower) >= X then f(a) >= f(lower) >= X, so there is no a such that f(a) < X
- // f(lower) >= X means rows cannot match
- int cmp = lit.comparator().compare(lower, lit.value());
- if (cmp >= 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean ltEq(Bound<T> term, Literal<T> lit) {
- // all terms are null preserving. see #isNullPreserving(Bound)
- int id = term.ref().fieldId();
- if (containsNullsOnly(id) || containsNaNsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- T lower = lowerBound(term);
- if (null == lower || NaNUtil.isNaN(lower)) {
- // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more.
- return ROWS_MIGHT_MATCH;
- }
-
- // this also works for transforms that are order preserving:
- // if a transform f is order preserving, a < b means that f(a) <= f(b).
- // because lower <= a for all values of a in the file, f(lower) <= f(a).
- // when f(lower) > X then f(a) >= f(lower) > X, so there is no a such that f(a) <= X
- // f(lower) > X means rows cannot match
- int cmp = lit.comparator().compare(lower, lit.value());
- if (cmp > 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean gt(Bound<T> term, Literal<T> lit) {
- // all terms are null preserving. see #isNullPreserving(Bound)
- int id = term.ref().fieldId();
- if (containsNullsOnly(id) || containsNaNsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- T upper = upperBound(term);
- if (null == upper) {
- return ROWS_MIGHT_MATCH;
- }
-
- int cmp = lit.comparator().compare(upper, lit.value());
- if (cmp <= 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean gtEq(Bound<T> term, Literal<T> lit) {
- // all terms are null preserving. see #isNullPreserving(Bound)
- int id = term.ref().fieldId();
- if (containsNullsOnly(id) || containsNaNsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- T upper = upperBound(term);
- if (null == upper) {
- return ROWS_MIGHT_MATCH;
- }
-
- int cmp = lit.comparator().compare(upper, lit.value());
- if (cmp < 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean eq(Bound<T> term, Literal<T> lit) {
- // all terms are null preserving. see #isNullPreserving(Bound)
- int id = term.ref().fieldId();
- if (containsNullsOnly(id) || containsNaNsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- T lower = lowerBound(term);
- if (lower != null && !NaNUtil.isNaN(lower)) {
- int cmp = lit.comparator().compare(lower, lit.value());
- if (cmp > 0) {
- return ROWS_CANNOT_MATCH;
- }
- }
-
- T upper = upperBound(term);
- if (null == upper) {
- return ROWS_MIGHT_MATCH;
- }
-
- int cmp = lit.comparator().compare(upper, lit.value());
- if (cmp < 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean notEq(Bound<T> term, Literal<T> lit) {
- // because the bounds are not necessarily a min or max value, this cannot be answered using
- // them. notEq(col, X) with (X, Y) doesn't guarantee that X is a value in col.
- // However, when min == max and the file has no nulls or NaN values, we can safely prune
- // if that value equals the literal.
- T value = uniqueValue(term);
-
- if (value != null && lit.comparator().compare(value, lit.value()) == 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean in(Bound<T> term, Set<T> literalSet) {
- // all terms are null preserving. see #isNullPreserving(Bound)
- int id = term.ref().fieldId();
- if (containsNullsOnly(id) || containsNaNsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- Collection<T> literals = literalSet;
-
- if (literals.size() > IN_PREDICATE_LIMIT) {
- // skip evaluating the predicate if the number of values is too big
- return ROWS_MIGHT_MATCH;
- }
-
- T lower = lowerBound(term);
- if (null == lower || NaNUtil.isNaN(lower)) {
- // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more.
- return ROWS_MIGHT_MATCH;
- }
-
- literals =
- literals.stream()
- .filter(v -> ((BoundTerm<T>) term).comparator().compare(lower, v) <= 0)
- .collect(Collectors.toList());
- // if all values are less than lower bound, rows cannot match
- if (literals.isEmpty()) {
- return ROWS_CANNOT_MATCH;
- }
-
- T upper = upperBound(term);
- if (null == upper) {
- return ROWS_MIGHT_MATCH;
- }
-
- literals =
- literals.stream()
- .filter(v -> ((BoundTerm<T>) term).comparator().compare(upper, v) >= 0)
- .collect(Collectors.toList());
- // if remaining values are greater than upper bound, rows cannot match
- if (literals.isEmpty()) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean notIn(Bound<T> term, Set<T> literalSet) {
- // because the bounds are not necessarily a min or max value, this cannot be answered using
- // them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col.
- // However, when min == max and the file has no nulls or NaN values, we can safely prune
- // if that value is in the exclusion set.
- T value = uniqueValue(term);
-
- if (value != null && literalSet.contains(value)) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean startsWith(Bound<T> term, Literal<T> lit) {
- if (term instanceof BoundTransform
- && !((BoundTransform<?, ?>) term).transform().isIdentity()) {
- // truncate must be rewritten in binding. the result is either always or never compatible
- return ROWS_MIGHT_MATCH;
- }
-
- int id = term.ref().fieldId();
- if (containsNullsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- String prefix = (String) lit.value();
-
- Comparator<CharSequence> comparator = Comparators.charSequences();
-
- CharSequence lower = (CharSequence) lowerBound(term);
- if (null == lower) {
- return ROWS_MIGHT_MATCH;
- }
-
- // truncate lower bound so that its length in bytes is not greater than the length of prefix
- int length = Math.min(prefix.length(), lower.length());
- int cmp = comparator.compare(lower.subSequence(0, length), prefix);
- if (cmp > 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- CharSequence upper = (CharSequence) upperBound(term);
- if (null == upper) {
- return ROWS_MIGHT_MATCH;
- }
-
- // truncate upper bound so that its length in bytes is not greater than the length of prefix
- length = Math.min(prefix.length(), upper.length());
- cmp = comparator.compare(upper.subSequence(0, length), prefix);
- if (cmp < 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public <T> Boolean notStartsWith(Bound<T> term, Literal<T> lit) {
- // the only transforms that produce strings are truncate and identity, which work with this
- int id = term.ref().fieldId();
- if (mayContainNull(id)) {
- return ROWS_MIGHT_MATCH;
- }
-
- String prefix = (String) lit.value();
-
- Comparator<CharSequence> comparator = Comparators.charSequences();
-
- // notStartsWith will match unless all values must start with the prefix. This happens when
- // the lower and upper bounds both start with the prefix.
- CharSequence lower = (CharSequence) lowerBound(term);
- CharSequence upper = (CharSequence) upperBound(term);
- if (null == lower || null == upper) {
- return ROWS_MIGHT_MATCH;
- }
-
- // if lower is shorter than the prefix then lower doesn't start with the prefix
- if (lower.length() < prefix.length()) {
- return ROWS_MIGHT_MATCH;
- }
-
- int cmp = comparator.compare(lower.subSequence(0, prefix.length()), prefix);
- if (cmp == 0) {
- // if upper is shorter than the prefix then upper can't start with the prefix
- if (upper.length() < prefix.length()) {
- return ROWS_MIGHT_MATCH;
- }
-
- cmp = comparator.compare(upper.subSequence(0, prefix.length()), prefix);
- if (cmp == 0) {
- // both bounds match the prefix, so all rows must match the prefix and therefore do not
- // satisfy the predicate
- return ROWS_CANNOT_MATCH;
- }
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- private boolean mayContainNull(Integer id) {
+ protected boolean mayContainNull(int id) {
return nullCounts == null || !nullCounts.containsKey(id) || nullCounts.get(id) != 0;
}
- private boolean containsNullsOnly(Integer id) {
+ @Override
+ protected boolean containsNullsOnly(int id) {
return valueCounts != null
&& valueCounts.containsKey(id)
&& nullCounts != null
@@ -499,66 +110,21 @@ public class InclusiveMetricsEvaluator {
&& valueCounts.get(id) - nullCounts.get(id) == 0;
}
- private boolean containsNaNsOnly(Integer id) {
+ @Override
+ protected boolean mayContainNaN(int id) {
+ return nanCounts == null || !nanCounts.containsKey(id) || nanCounts.get(id) != 0;
+ }
+
+ @Override
+ protected boolean containsNaNsOnly(int id) {
return nanCounts != null
&& nanCounts.containsKey(id)
&& valueCounts != null
&& nanCounts.get(id).equals(valueCounts.get(id));
}
- /**
- * Returns the column's single value if all rows contain the same value. Defined as a column
- * with no nulls, no NaNs, and lower bound equals upper bound. Returns null otherwise.
- */
- private <T> T uniqueValue(Bound<T> term) {
- int id = term.ref().fieldId();
- if (mayContainNull(id)) {
- return null;
- }
-
- T lower = lowerBound(term);
- T upper = upperBound(term);
-
- if (lower == null || upper == null || NaNUtil.isNaN(lower) || NaNUtil.isNaN(upper)) {
- return null;
- }
-
- if (nanCounts != null && nanCounts.containsKey(id) && nanCounts.get(id) != 0) {
- return null;
- }
-
- if (!lower.equals(upper)) {
- return null;
- }
-
- return lower;
- }
-
- private <T> T lowerBound(Bound<T> term) {
- if (term instanceof BoundReference) {
- return parseLowerBound((BoundReference<T>) term);
- } else if (term instanceof BoundTransform) {
- return transformLowerBound((BoundTransform<?, T>) term);
- } else if (term instanceof BoundExtract) {
- return extractLowerBound((BoundExtract<T>) term);
- } else {
- return null;
- }
- }
-
- private <T> T upperBound(Bound<T> term) {
- if (term instanceof BoundReference) {
- return parseUpperBound((BoundReference<T>) term);
- } else if (term instanceof BoundTransform) {
- return transformUpperBound((BoundTransform<?, T>) term);
- } else if (term instanceof BoundExtract) {
- return extractUpperBound((BoundExtract<T>) term);
- } else {
- return null;
- }
- }
-
- private <T> T parseLowerBound(BoundReference<T> ref) {
+ @Override
+ protected <T> T lowerBound(BoundReference<T> ref) {
int id = ref.fieldId();
if (lowerBounds != null && lowerBounds.containsKey(id)) {
return Conversions.fromByteBuffer(ref.ref().type(), lowerBounds.get(id));
@@ -567,7 +133,8 @@ public class InclusiveMetricsEvaluator {
return null;
}
- private <T> T parseUpperBound(BoundReference<T> ref) {
+ @Override
+ protected <T> T upperBound(BoundReference<T> ref) {
int id = ref.fieldId();
if (upperBounds != null && upperBounds.containsKey(id)) {
return Conversions.fromByteBuffer(ref.ref().type(), upperBounds.get(id));
@@ -576,27 +143,8 @@ public class InclusiveMetricsEvaluator {
return null;
}
- private <S, T> T transformLowerBound(BoundTransform<S, T> boundTransform) {
- Transform<S, T> transform = boundTransform.transform();
- if (transform.preservesOrder()) {
- S lower = parseLowerBound(boundTransform.ref());
- return boundTransform.transform().bind(boundTransform.ref().type()).apply(lower);
- }
-
- return null;
- }
-
- private <S, T> T transformUpperBound(BoundTransform<S, T> boundTransform) {
- Transform<S, T> transform = boundTransform.transform();
- if (transform.preservesOrder()) {
- S upper = parseUpperBound(boundTransform.ref());
- return boundTransform.transform().bind(boundTransform.ref().type()).apply(upper);
- }
-
- return null;
- }
-
- private <T> T extractLowerBound(BoundExtract<T> bound) {
+ @Override
+ protected <T> T extractLowerBound(BoundExtract<T> bound) {
int id = bound.ref().fieldId();
if (lowerBounds != null && lowerBounds.containsKey(id)) {
VariantObject fieldLowerBounds = parseBounds(lowerBounds.get(id));
@@ -606,7 +154,8 @@ public class InclusiveMetricsEvaluator {
return null;
}
- private <T> T extractUpperBound(BoundExtract<T> bound) {
+ @Override
+ protected <T> T extractUpperBound(BoundExtract<T> bound) {
int id = bound.ref().fieldId();
if (upperBounds != null && upperBounds.containsKey(id)) {
VariantObject fieldUpperBounds = parseBounds(upperBounds.get(id));
@@ -615,20 +164,6 @@ public class InclusiveMetricsEvaluator {
return null;
}
-
- /** Returns true if the expression term produces a non-null value for non-null input. */
- private boolean isNonNullPreserving(Bound<?> term) {
- if (term instanceof BoundReference) {
- return true;
- } else if (term instanceof BoundTransform<?, ?>) {
- // if the transform preserves order, then non-null values are mapped to non-null
- return ((BoundTransform<?, ?>) term).transform().preservesOrder();
- }
-
- // a non-null variant does not necessarily contain a specific field
- // and unknown bound terms are not non-null preserving
- return false;
- }
}
private static VariantObject parseBounds(ByteBuffer buffer) {
diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java
index 5f0ca2659f..8d652df0df 100644
--- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java
+++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java
@@ -41,7 +41,9 @@ import static org.apache.iceberg.types.Types.NestedField.required;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import java.nio.ByteBuffer;
import java.util.List;
+import java.util.Map;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.Schema;
import org.apache.iceberg.TestHelpers.Row;
@@ -971,6 +973,46 @@ public class TestInclusiveMetricsEvaluator {
.isFalse();
}
+ @Test
+ public void testNotEqSingleValueWithoutNaN() {
+ Schema schema = new Schema(required(1, "f", Types.FloatType.get()));
+ Map<Integer, ByteBuffer> bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f));
+ DataFile singleValueFile =
+ new TestDataFile(
+ "single_value_file.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(1, 10L),
+ ImmutableMap.of(1, 0L),
+ ImmutableMap.of(1, 0L),
+ bound,
+ bound);
+
+ assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile))
+ .as("Should skip: file contains no values not equal to 1.0")
+ .isFalse();
+ }
+
+ @Test
+ public void testNotEqSingleValueWithNaN() {
+ Schema schema = new Schema(required(1, "f", Types.FloatType.get()));
+ Map<Integer, ByteBuffer> bound = ImmutableMap.of(1, toByteBuffer(Types.FloatType.get(), 1.0f));
+ DataFile singleValueFile =
+ new TestDataFile(
+ "single_value_file.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(1, 10L),
+ ImmutableMap.of(1, 0L),
+ ImmutableMap.of(1, 1L), // contains a NaN value
+ bound,
+ bound);
+
+ assertThat(new InclusiveMetricsEvaluator(schema, notEqual("f", 1.0f)).eval(singleValueFile))
+ .as("Should read: file contains a NaN value not equal to 1.0")
+ .isTrue();
+ }
+
@Test
public void testNotEqWithSingleValue() {
DataFile rangeOfValues =
@@ -997,7 +1039,7 @@ public class TestInclusiveMetricsEvaluator {
10,
ImmutableMap.of(3, 10L),
ImmutableMap.of(3, 0L),
- ImmutableMap.of(3, 0L),
+ null,
ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")),
ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")));
@@ -1020,7 +1062,7 @@ public class TestInclusiveMetricsEvaluator {
10,
ImmutableMap.of(3, 10L),
ImmutableMap.of(3, 2L),
- ImmutableMap.of(3, 0L),
+ null,
ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")),
ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")));