You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by la...@apache.org on 2026/10/06 18:56:47 UTC
(impala) branch master updated (d6638f0d9 -> 346901c75)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from d6638f0d9 IMPALA-15412: Fix Formatting of error_description in WWW-Authenticate Header
new ef5d7c4a0 IMPALA-15371: Support IS [NOT] NULL on VARIANT columns
new 346901c75 IMPALA-11917: Update LLVM CPU attribute whitelists for LLVM 22
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/codegen/llvm-codegen-test.cc | 16 ++--
be/src/codegen/llvm-codegen.cc | 18 ++---
be/src/exprs/is-null-predicate-ir.cc | 3 +
.../apache/impala/analysis/IsNullPredicate.java | 9 +--
.../impala/analysis/TupleIsNullPredicate.java | 5 +-
.../main/java/org/apache/impala/catalog/Type.java | 13 ++++
.../org/apache/impala/planner/HdfsScanNode.java | 2 +
.../apache/impala/analysis/AnalyzeExprsTest.java | 9 +++
.../queries/QueryTest/iceberg-v3-variant.test | 87 ++++++++++++++++++++--
9 files changed, 132 insertions(+), 30 deletions(-)
(impala) 01/02: IMPALA-15371: Support IS [NOT] NULL on VARIANT columns
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ef5d7c4a02ad4577a1227340e3051b325a146544
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Thu Sep 17 20:13:09 2026 +0200
IMPALA-15371: Support IS [NOT] NULL on VARIANT columns
'v IS NULL' and 'v IS NOT NULL' were rejected during analysis for
VARIANT columns. The reason was that there was no is_null_pred()
builtin for VARIANT: the builtins are registered for the types of
Type.getSupportedTypes(), which are all scalar, and when VARIANT read
support was added a VARIANT had no expression ABI yet. So the check
only turned a "no matching function" error into a clearer message.
Since IMPALA-15057 a VARIANT is a self-contained value (VariantVal) at
the UDF boundary, just like the scalar types, therefore the builtins
can simply be instantiated for it.
Only SQL NULL is NULL. A variant null in the column is a regular value,
so 'v IS NULL' is false for it. Note that variant_get() does return SQL
NULL when there is a variant null at the path, hence
variant_get(v, '$') IS NULL can differ from v IS NULL.
The predicate is pushed down to Iceberg the same way as for the other
types. This is safe: Iceberg only uses it to prune whole files by their
NULL counts, which it derives from the required metadata column of the
VARIANT, and it never replaces the evaluation of the predicate.
Testing:
* FE: extended AnalyzeExprsTest.TestIsNullPredicates
* e2e tests added
Change-Id: Id0b17821f6c8a988abf93494b7dbeb8f6f7db216
Assisted-by: Claude Fable 5.1 (Claude Code)
Reviewed-on: http://gerrit.cloudera.org:8080/24881
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exprs/is-null-predicate-ir.cc | 3 +
.../apache/impala/analysis/IsNullPredicate.java | 9 +--
.../impala/analysis/TupleIsNullPredicate.java | 5 +-
.../main/java/org/apache/impala/catalog/Type.java | 13 ++++
.../org/apache/impala/planner/HdfsScanNode.java | 2 +
.../apache/impala/analysis/AnalyzeExprsTest.java | 9 +++
.../queries/QueryTest/iceberg-v3-variant.test | 87 ++++++++++++++++++++--
7 files changed, 112 insertions(+), 16 deletions(-)
diff --git a/be/src/exprs/is-null-predicate-ir.cc b/be/src/exprs/is-null-predicate-ir.cc
index 97b752d6a..1763516df 100644
--- a/be/src/exprs/is-null-predicate-ir.cc
+++ b/be/src/exprs/is-null-predicate-ir.cc
@@ -17,6 +17,7 @@
#include "exprs/is-null-predicate.h"
#include "udf/udf.h"
+#include "udf/udf-internal.h"
using namespace impala_udf;
@@ -43,6 +44,7 @@ template BooleanVal IsNullPredicate::IsNull(FunctionContext*, const DoubleVal&);
template BooleanVal IsNullPredicate::IsNull(FunctionContext*, const StringVal&);
template BooleanVal IsNullPredicate::IsNull(FunctionContext*, const TimestampVal&);
template BooleanVal IsNullPredicate::IsNull(FunctionContext*, const DecimalVal&);
+template BooleanVal IsNullPredicate::IsNull(FunctionContext*, const VariantVal&);
template BooleanVal IsNullPredicate::IsNotNull(FunctionContext*, const BooleanVal&);
template BooleanVal IsNullPredicate::IsNotNull(FunctionContext*, const TinyIntVal&);
@@ -55,5 +57,6 @@ template BooleanVal IsNullPredicate::IsNotNull(FunctionContext*, const DoubleVal
template BooleanVal IsNullPredicate::IsNotNull(FunctionContext*, const StringVal&);
template BooleanVal IsNullPredicate::IsNotNull(FunctionContext*, const TimestampVal&);
template BooleanVal IsNullPredicate::IsNotNull(FunctionContext*, const DecimalVal&);
+template BooleanVal IsNullPredicate::IsNotNull(FunctionContext*, const VariantVal&);
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/IsNullPredicate.java b/fe/src/main/java/org/apache/impala/analysis/IsNullPredicate.java
index 641361d9c..40e091dd6 100644
--- a/fe/src/main/java/org/apache/impala/analysis/IsNullPredicate.java
+++ b/fe/src/main/java/org/apache/impala/analysis/IsNullPredicate.java
@@ -58,7 +58,8 @@ public class IsNullPredicate extends Predicate {
public boolean isNotNull() { return isNotNull_; }
public static void initBuiltins(Db db) {
- for (Type t: Type.getSupportedTypes()) {
+ // For VARIANT, a variant null is a regular value, only SQL NULLs are NULL.
+ for (Type t: Type.getIsNullPredicateTypes()) {
if (t.isNull()) continue;
String isNullSymbol;
if (t.isBoolean()) {
@@ -128,11 +129,9 @@ public class IsNullPredicate extends Predicate {
uncheckedCastChild(ScalarType.BOOLEAN, 0);
}
- if (getChild(0).getType().isComplexOrVariantType()) {
- String typeStr =
- getChild(0).getType().isVariantType() ? "VARIANT type" : "complex types";
+ if (getChild(0).getType().isComplexType()) {
String errorMsg = (isNotNull_ ? "IS NOT NULL" : "IS NULL") +
- " predicate does not support " + typeStr + ": ";
+ " predicate does not support complex types: ";
throw new AnalysisException(errorMsg + toSqlImpl());
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/TupleIsNullPredicate.java b/fe/src/main/java/org/apache/impala/analysis/TupleIsNullPredicate.java
index 16463bc11..e8e4eb278 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TupleIsNullPredicate.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TupleIsNullPredicate.java
@@ -178,9 +178,8 @@ public class TupleIsNullPredicate extends Predicate {
if (expr.getType().isComplexOrVariantType()) {
// Complex/variant exprs never need NULL wrapping. A VARIANT expr can now be a
// SlotRef (scan pass-through) or a function result (e.g. variant_get()), so we no
- // longer assert SlotRef-only here. Without this early return a variant expr (e.g.
- // through an outer-join inline view) would fall through to IsNullPredicate analysis
- // below, which rejects variant types and throws (swallowed by analyzeNoThrow).
+ // longer assert SlotRef-only here. The functions that return a VARIANT return NULL
+ // for a NULL input, and there is no if() that could wrap a VARIANT anyway.
return false;
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/Type.java b/fe/src/main/java/org/apache/impala/catalog/Type.java
index dd10c23f2..421cc74df 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Type.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Type.java
@@ -83,6 +83,7 @@ public abstract class Type {
private static List<ScalarType> integerTypes;
private static List<ScalarType> numericTypes;
private static List<ScalarType> supportedTypes;
+ private static List<Type> isNullPredicateTypes;
private static List<ScalarType> unsupportedTypes;
static {
@@ -120,6 +121,9 @@ public abstract class Type {
supportedTypes.add(UUID);
supportedTypes.add(GEOMETRY);
+ isNullPredicateTypes = new ArrayList<>(supportedTypes);
+ isNullPredicateTypes.add(VARIANT);
+
unsupportedTypes = new ArrayList<>();
unsupportedTypes.add(DATETIME);
}
@@ -133,6 +137,15 @@ public abstract class Type {
public static List<ScalarType> getSupportedTypes() {
return supportedTypes;
}
+ /**
+ * Returns the argument types of the IS [NOT] NULL builtins: the supported scalar types
+ * and VARIANT. VARIANT is not a scalar type, but unlike the complex types, a VARIANT
+ * value is passed to builtins as a self-contained value (VariantVal, see
+ * Function.getUdfType()), so it can be NULL-checked like a scalar value.
+ */
+ public static List<Type> getIsNullPredicateTypes() {
+ return isNullPredicateTypes;
+ }
public static List<ScalarType> getUnsupportedTypes() {
return unsupportedTypes;
}
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index 716f0eb50..77d48aef8 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -895,6 +895,8 @@ public class HdfsScanNode extends ScanNode {
Preconditions.checkState(slotRef.getDesc().isScanSlot());
// Skip the slot ref if it refers to an array's "pos" field.
if (slotRef.getDesc().isArrayPosRef()) return;
+ // Only primitive types can be pushed down to the ORC reader.
+ if (slotRef.getDesc().getType().isVariantType()) return;
addStatsOriginalConjunct(slotRef.getDesc().getParent(), isNullPred);
SlotDescriptor slotDesc = analyzer.getDescTbl().copySlotDescriptor(statsTuple_,
slotRef.getDesc());
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
index c8b0598cb..475238209 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
@@ -695,6 +695,15 @@ public class AnalyzeExprsTest extends AnalyzerTest {
AnalyzesOk("select * from functional.alltypes where string_col is not null");
AnalyzesOk("select * from functional.binary_tbl where binary_col is null");
AnalyzesOk("select * from functional.alltypes where null is not null");
+ // VARIANT values can be NULL-checked, also when they are returned by a function.
+ AnalyzesOk("select id from functional_parquet.trino_variant where v is null");
+ AnalyzesOk("select v is not null from functional_parquet.trino_variant");
+ AnalyzesOk("select id from functional_parquet.trino_variant " +
+ "where variant_get(v, '$.name') is not null");
+ AnalyzesOk("select if(v is null, 0, 1), nvl2(v, 1, 0), " +
+ "case when v is null then 'null' else 'not null' end, " +
+ "count(case when v is null then 1 else null end) " +
+ "from functional_parquet.trino_variant group by 1, 2, 3");
AnalysisError("select 1 from functional.allcomplextypes where int_map_col is null",
"IS NULL predicate does not support complex types: int_map_col IS NULL");
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-variant.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-variant.test
index 99529bb76..9de4a86ae 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-variant.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-variant.test
@@ -354,8 +354,85 @@ INT, STRING
aggregation(SUM, TotalMergesPerformed)> 0
====
---- QUERY
-# Negative cases: VARIANT is not comparable/orderable/hashable and can only be a
-# pass-through SlotRef, so these must raise a clean AnalysisException (never crash/NPE).
+# IS NULL on a variant.
+SELECT id FROM functional_parquet.trino_variant WHERE v IS NULL;
+---- RESULTS
+0
+---- TYPES
+INT
+====
+---- QUERY
+SELECT count(*) FROM functional_parquet.trino_variant WHERE v IS NOT NULL;
+---- RESULTS
+54
+---- TYPES
+BIGINT
+====
+---- QUERY
+# A variant null (id 1) is a regular value, it is not SQL NULL.
+SELECT id, v, v IS NULL, v IS NOT NULL FROM functional_parquet.trino_variant
+WHERE id IN (0, 1, 2) ORDER BY id;
+---- RESULTS
+0,'NULL',true,false
+1,'null',false,true
+2,'true',false,true
+---- TYPES
+INT, STRING, BOOLEAN, BOOLEAN
+====
+---- QUERY
+# IS NULL in aggregates and in conditional functions.
+SELECT nvl2(v, 'not null', 'null'), count(*),
+ count(CASE WHEN v IS NULL THEN 1 ELSE NULL END), sum(if(v IS NOT NULL, 1, 0))
+FROM functional_parquet.trino_variant GROUP BY 1 ORDER BY 1;
+---- RESULTS
+'not null',54,0,54
+'null',1,1,0
+---- TYPES
+STRING, BIGINT, BIGINT, BIGINT
+====
+---- QUERY
+# IS NULL on the variant of the nullable side of an outer join.
+SELECT a.id, b.v IS NULL
+FROM functional_parquet.trino_variant a
+ LEFT JOIN functional_parquet.trino_variant b ON a.id = b.id + 1
+WHERE a.id < 3 ORDER BY a.id;
+---- RESULTS
+0,true
+1,true
+2,false
+---- TYPES
+INT, BOOLEAN
+====
+---- QUERY
+# IS NULL evaluated on top of TOP-N and UNION ALL, i.e. on materialized variants.
+SELECT id, v IS NULL FROM (
+ SELECT id, v FROM functional_parquet.trino_variant WHERE id < 2
+ UNION ALL
+ (SELECT id + 100, v FROM functional_parquet.trino_variant ORDER BY 1 LIMIT 2)) u
+ORDER BY id;
+---- RESULTS
+0,true
+1,false
+100,true
+101,false
+---- TYPES
+BIGINT, BOOLEAN
+====
+---- QUERY
+# A sub-value returned by variant_get() is NULL if the path is missing ($.nope), and also
+# if there is a variant null at the path ($.a, or $ of id 1), unlike the variant null in
+# the column itself.
+SELECT id, variant_get(v, '$') IS NULL, variant_get(v, '$.a') IS NULL,
+ variant_get(v, '$.d') IS NULL, variant_get(v, '$.nope') IS NULL, v IS NULL
+FROM functional_parquet.trino_variant WHERE id IN (1, 40) ORDER BY id;
+---- RESULTS
+1,true,true,true,true,false
+40,false,true,false,true,false
+---- TYPES
+INT, BOOLEAN, BOOLEAN, BOOLEAN, BOOLEAN, BOOLEAN
+====
+---- QUERY
+# Negative cases: VARIANT is not comparable/orderable/hashable.
# ORDER BY a variant.
SELECT id FROM functional_parquet.trino_variant ORDER BY v;
---- CATCH
@@ -416,12 +493,6 @@ SELECT count(DISTINCT v) FROM functional_parquet.trino_variant;
DISTINCT aggregate parameter 'v' with type 'VARIANT' is not supported.
====
---- QUERY
-# IS NULL on a variant.
-SELECT id FROM functional_parquet.trino_variant WHERE v IS NULL;
----- CATCH
-IS NULL predicate does not support VARIANT type: v IS NULL
-====
----- QUERY
# INSERT into a table that has a variant column.
INSERT INTO functional_parquet.trino_variant SELECT id, descr, v FROM functional_parquet.trino_variant;
---- CATCH
(impala) 02/02: IMPALA-11917: Update LLVM CPU attribute whitelists for LLVM 22
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 346901c75e147e3a414f94bce85260b6db2f3bd4
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Thu Oct 1 13:38:45 2026 -0700
IMPALA-11917: Update LLVM CPU attribute whitelists for LLVM 22
LLVM 22 host detection reports cx8, fxsr and sahf, which were not
whitelisted and so were explicitly disabled. Disabling cx8 also
disables cx16 and limits atomics to 32 bits on x86-64. Add them to
the x86 whitelist and drop the hle and smap entries, which are not
LLVM features. Also add attrs for future use (clflushopt, clwb, pku,
sha, vaes, vpclmulqdq, xsavec, xsaves).
On AArch64, LLVM 22 no longer reports 'crypto'; replace it with aes
and sha2, and add lse, rand and sm4. Extend CpuAttrWhitelist to check
that the baseline attrs remain enabled.
Assisted-by: Claude Sonnet 5.5 (GitHub Copilot)
Change-Id: Iddd147d466b40052852ad6e136ee3ddfd59c46ac
Reviewed-on: http://gerrit.cloudera.org:8080/24999
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/codegen/llvm-codegen-test.cc | 16 +++++++++++-----
be/src/codegen/llvm-codegen.cc | 18 +++++++++---------
2 files changed, 20 insertions(+), 14 deletions(-)
diff --git a/be/src/codegen/llvm-codegen-test.cc b/be/src/codegen/llvm-codegen-test.cc
index 09fcd7fb9..df3e82018 100644
--- a/be/src/codegen/llvm-codegen-test.cc
+++ b/be/src/codegen/llvm-codegen-test.cc
@@ -587,11 +587,17 @@ TEST_F(LlvmCodeGenTest, CpuAttrWhitelist) {
LlvmCodeGen::ApplyCpuAttrWhitelist(
{"+dummy1", "+dummy2", "-dummy3", "+dummy4", "+sse2", "-lzcnt"}));
// IMPALA-6291: Test that all AVX512 attributes are disabled.
- vector<string> avx512_attrs;
- EXPECT_EQ(std::unordered_set<string>({"-avx512ifma", "-avx512dqavx512er", "-avx512f",
- "-avx512bw", "-avx512vl", "-avx512cd", "-avx512vbmi", "-avx512pf"}),
- LlvmCodeGen::ApplyCpuAttrWhitelist({"+avx512ifma", "+avx512dqavx512er", "+avx512f",
- "+avx512bw", "+avx512vl", "+avx512cd", "+avx512vbmi", "+avx512pf"}));
+ EXPECT_EQ(std::unordered_set<string>({
+ "-avx512ifma", "-avx512dq", "-avx512er", "-avx512f", "-avx512bw",
+ "-avx512vl", "-avx512cd", "-avx512vbmi", "-avx512pf"}),
+ LlvmCodeGen::ApplyCpuAttrWhitelist({
+ "+avx512ifma", "+avx512dq", "+avx512er", "+avx512f", "+avx512bw",
+ "+avx512vl", "+avx512cd", "+avx512vbmi", "+avx512pf"}));
+ // cx8 must stay enabled: disabling it also disables cx16 and 64-bit atomics.
+ const std::unordered_set<string> baseline = IS_AARCH64 ?
+ std::unordered_set<string>({"+crc", "+lse", "+neon"}) :
+ std::unordered_set<string>({"+cx8", "+cx16", "+fxsr", "+sahf", "+crc32"});
+ EXPECT_EQ(baseline, LlvmCodeGen::ApplyCpuAttrWhitelist(baseline));
}
// Test that exercises the code path that deletes non-finalized methods before it
diff --git a/be/src/codegen/llvm-codegen.cc b/be/src/codegen/llvm-codegen.cc
index 182c0c802..833cea589 100644
--- a/be/src/codegen/llvm-codegen.cc
+++ b/be/src/codegen/llvm-codegen.cc
@@ -111,20 +111,20 @@ DEFINE_string(asm_module_dir, "",
"if set, saves disassembly for generated IR modules to the specified directory.");
DECLARE_string(local_library_dir);
// IMPALA-6291: AVX-512 and other CPU attrs the community doesn't routinely test are
-// disabled. AVX-512 is affected by known bugs in LLVM 3.9.1. The following attrs that
-// exist in LLVM 3.9.1 are disabled: avx512bw,avx512cd,avx512dq,avx512er,avx512f,
-// avx512ifma,avx512pf,avx512vbmi,avx512vl,clflushopt,clwb,fma4,mwaitx.1.2,pcommit,pku,
-// prefetchwt1,sgx,sha,sse4a,tbm,xop,xsavec,xsaves. If new attrs are added to LLVM,
-// they will be disabled until added to this whitelist.
+// disabled. Any detected attr not listed here is explicitly disabled, overriding the
+// defaults implied by the CPU name, so baseline attrs that LLVM detects (e.g. cx8,
+// which cx16 implies and 64-bit atomics need) must be listed. Attrs deliberately left
+// out: AVX-512, gfni, avxvnni*, amx-*, avx10.*, APX, sse4a, sgx, sha3 and SVE.
#ifdef __aarch64__
-DEFINE_string_hidden(llvm_cpu_attr_whitelist, "crc,neon,fp-armv8,crypto",
+DEFINE_string_hidden(llvm_cpu_attr_whitelist, "aes,crc,fp-armv8,lse,neon,rand,sha2,sm4",
"(Experimental) a comma-separated list of LLVM CPU attribute flags that are enabled "
"for runtime code generation. This flag is provided to enable additional LLVM CPU "
"attribute flags for testing.");
#else
-DEFINE_string_hidden(llvm_cpu_attr_whitelist, "64bit,adx,aes,avx,avx2,bmi,bmi2,cmov,cx16,"
- "f16c,fma,fsgsbase,hle,invpcid,lzcnt,mmx,movbe,pclmul,popcnt,prfchw,rdrnd,rdseed,rtm,"
- "smap,sse,sse2,sse3,sse4.1,sse4.2,ssse3,xsave,xsaveopt,crc32",
+DEFINE_string_hidden(llvm_cpu_attr_whitelist, "64bit,adx,aes,avx,avx2,bmi,bmi2,"
+ "clflushopt,clwb,cmov,crc32,cx16,cx8,f16c,fma,fsgsbase,fxsr,invpcid,lzcnt,mmx,"
+ "movbe,pclmul,pku,popcnt,prfchw,rdrnd,rdseed,rtm,sahf,sha,sse,sse2,sse3,sse4.1,"
+ "sse4.2,ssse3,vaes,vpclmulqdq,xsave,xsavec,xsaveopt,xsaves",
"(Experimental) a comma-separated list of LLVM CPU attribute flags that are enabled "
"for runtime code generation. The default flags are a known-good set that are "
"routinely tested. This flag is provided to enable additional LLVM CPU attribute "