You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by cs...@apache.org on 2026/07/03 11:23:06 UTC
(impala) 01/03: IMPALA-15130: Reject column masking on complex-typed columns
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit af1928236e53eef7e372168e47403bc67e05549a
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Wed Jul 1 15:53:42 2026 +0200
IMPALA-15130: Reject column masking on complex-typed columns
A Ranger column-masking policy on a top-level complex-typed column
(ARRAY/MAP/STRUCT) cannot be applied: the mask transforms are scalar
and there is no way to express a masked complex value.
TableMask.createColumnMaskStmt() previously ignored such masks and
returned the column unmasked, leaking data the policy was meant to hide.
The InlineViewRef guard that was supposed to reject this ("Column
masking is not supported for complex types") was dead code as
TableMask.hasComplexColumnMask_ was never assigned.
Fail closed instead: a mask on a top-level complex column now throws an
AnalysisException. Masks on nested struct fields (e.g. nested_struct.a)
are still ignored, matching Hive, which only considers top-level columns
and so does not recognize such policies either.
Also removes the dead hasComplexColumnMask_ field/getter and the
unreachable InlineViewRef check.
Testing:
- New FE test AuthorizationStmtTest.testColumnMaskingOnComplexColumn:
a mask on a top-level ARRAY, MAP and STRUCT column is rejected, while a
mask on a nested struct field is still ignored.
- test_column_masking_with_structs_in_select_list keeps the masks on the
complex columns (tiny_struct, int_array_array, int_map_array) and the
ranger_column_masking_struct_in_select_list.test goldens now assert
that selecting those columns fails with the new error. This shows the
end-to-end behaviour next to the related masking tests in that file.
- test_column_masking: dropped only the mask on the top-level int_array
column (that column is exercised as a nested collection reference all
over ranger_column_masking_complex_types.test and would now error). The
mask on the nested field nested_struct.a is kept, since nested-field
masks are still ignored (Hive parity), so those goldens are unchanged.
Change-Id: Idddacf0b40abcaa18a5764ff617179fe898886c9
Assisted-by: Claude Opus 4.8 (1M context) <no...@anthropic.com>
Reviewed-on: http://gerrit.cloudera.org:8080/24530
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/analysis/InlineViewRef.java | 3 --
.../org/apache/impala/authorization/TableMask.java | 19 ++++---
.../authorization/AuthorizationStmtTest.java | 63 ++++++++++++++++++++++
...anger_column_masking_struct_in_select_list.test | 41 ++++++--------
tests/authorization/test_ranger.py | 15 +++---
5 files changed, 99 insertions(+), 42 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
index 08243fb1b..97bd008df 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
@@ -174,9 +174,6 @@ public class InlineViewRef extends TableRef {
boolean isHidden = col.isVirtual();
items.add(new SelectListItem(maskExpr, /*alias*/ col.getName(), isHidden));
}
- if (tableMask.hasComplexColumnMask()) {
- throw new AnalysisException("Column masking is not supported for complex types");
- }
if (columns.isEmpty()) {
// No columns so use "SELECT 1 FROM tbl" to make a valid statement.
items.add(new SelectListItem(NumericLiteral.create(1), /*alias*/null));
diff --git a/fe/src/main/java/org/apache/impala/authorization/TableMask.java b/fe/src/main/java/org/apache/impala/authorization/TableMask.java
index de3289792..0b4cd4c58 100644
--- a/fe/src/main/java/org/apache/impala/authorization/TableMask.java
+++ b/fe/src/main/java/org/apache/impala/authorization/TableMask.java
@@ -47,8 +47,6 @@ public class TableMask {
private final List<String> requiredColumnNames_;
private final User user_;
- private boolean hasComplexColumnMask_;
-
public TableMask(AuthorizationChecker authzChecker, String dbName, String tableName,
List<Column> requiredColumns, User user) {
this.authChecker_ = Preconditions.checkNotNull(authzChecker);
@@ -61,7 +59,6 @@ public class TableMask {
}
public List<Column> getRequiredColumns() { return requiredColumns_; }
- public boolean hasComplexColumnMask () { return hasComplexColumnMask_; }
/**
* Returns whether the table/view has column masking or row filtering policies.
@@ -90,13 +87,23 @@ public class TableMask {
if (maskedValue == null || maskedValue.equals(colName)) { // Don't need masking.
return null;
}
- if (colType.isComplexType() || colName.contains(".")) {
- // Ignore column masks on complex types or their children.
+ if (colName.contains(".")) {
+ // Ignore column masks on nested fields of a complex column. Hive only considers
+ // top-level columns for masking, so such policies are not recognized there either;
+ // keep that parity here rather than erroring.
// TODO: RANGER-3525: Clarify handling of column masks on nested types
- LOG.warn("Ignoring column mask on complex type {}.{}: {} => {}",
+ LOG.warn("Ignoring column mask on nested field {}.{}: {} => {}",
dbName_, tableName_, colName, maskedValue);
return null;
}
+ if (colType.isComplexType()) {
+ // A column-masking policy on a top-level complex-typed column cannot be applied:
+ // the mask transforms are scalar and there is no way to express a masked
+ // ARRAY/MAP/STRUCT value.
+ throw new AnalysisException(String.format(
+ "Column masking is not supported for complex type column '%s.%s.%s' (%s)",
+ dbName_, tableName_, colName, colType.toSql()));
+ }
SelectStmt maskStmt = (SelectStmt) Parser.parse(
String.format("SELECT CAST(%s AS %s)", maskedValue, colType));
if (maskStmt.getSelectList().getItems().size() != 1 || maskStmt.hasGroupByClause()
diff --git a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
index 5eeb07c06..197454cf4 100644
--- a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
@@ -55,6 +55,7 @@ import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* This class contains authorization tests for SQL statements.
@@ -3369,6 +3370,68 @@ public class AuthorizationStmtTest extends AuthorizationTestBase {
}
}
+ /**
+ * IMPALA-15130: A column-masking policy on a top-level complex-typed column
+ * (ARRAY/MAP/STRUCT) cannot be applied, so Impala must reject the query
+ * rather than silently returning the column unmasked. A policy on a nested struct
+ * field is ignored instead (parity with Hive, which only considers top-level columns).
+ */
+ @Test
+ public void testColumnMaskingOnComplexColumn() throws ImpalaException {
+ String arrPolicy = "complextypestbl_int_array_mask";
+ String mapPolicy = "complextypestbl_int_map_mask";
+ String structPolicy = "complextypestbl_nested_struct_mask";
+ String nestedPolicy = "complextypestbl_nested_field_mask";
+ AnalysisContext ctx = createAnalysisCtx(authzFactory_, user_.getName());
+ try {
+ // A mask on a top-level complex column is rejected regardless of the complex kind:
+ // an ARRAY column ...
+ expectComplexColumnMaskRejected(ctx, arrPolicy, "functional_parquet", "int_array",
+ "select int_array from functional_parquet.complextypestbl");
+ // ... a MAP column ...
+ expectComplexColumnMaskRejected(ctx, mapPolicy, "functional_parquet", "int_map",
+ "select int_map from functional_parquet.complextypestbl");
+ // ... and a STRUCT column (functional_orc_def supports reading structs directly).
+ expectComplexColumnMaskRejected(ctx, structPolicy, "functional_orc_def",
+ "nested_struct",
+ "select nested_struct from functional_orc_def.complextypestbl");
+
+ // A mask on a nested struct field is ignored (parity with Hive), so the query still
+ // analyzes.
+ createColumnMaskingPolicy(nestedPolicy, "functional_parquet", "complextypestbl",
+ "nested_struct.a", user_.getShortName(), "CUSTOM", "100 * {col}");
+ rangerImpalaPlugin_.refreshPoliciesAndTags();
+ authorize(ctx, "select id, nested_struct.a from functional_parquet.complextypestbl")
+ .ok(onServer(TPrivilegeLevel.ALL));
+ } finally {
+ try { deleteRangerPolicy(arrPolicy); } catch (Exception e) { /* may not exist */ }
+ try { deleteRangerPolicy(mapPolicy); } catch (Exception e) { /* ditto */ }
+ try { deleteRangerPolicy(structPolicy); } catch (Exception e) { /* ditto */ }
+ try { deleteRangerPolicy(nestedPolicy); } catch (Exception e) { /* ditto */ }
+ }
+ }
+
+ /**
+ * Adds a MASK_NULL policy on the given top-level complex column of 'complextypestbl'
+ * and asserts that a query referencing it is rejected during analysis. The mask is
+ * applied while building the table-masking view, so it surfaces as an AnalysisException
+ * before authorization is evaluated.
+ */
+ private void expectComplexColumnMaskRejected(AnalysisContext ctx, String policyName,
+ String db, String col, String query) throws ImpalaException {
+ createColumnMaskingPolicy(policyName, db, "complextypestbl", col,
+ user_.getShortName(), "MASK_NULL", /*maskExpr*/ null);
+ rangerImpalaPlugin_.refreshPoliciesAndTags();
+ String err = "Column masking is not supported for complex type column '" + db
+ + ".complextypestbl." + col + "'";
+ try {
+ authorize(ctx, query).error(err, onServer(TPrivilegeLevel.ALL));
+ fail("Expected masking on a complex column to be rejected");
+ } catch (AnalysisException e) {
+ assertTrue(e.getMessage(), e.getMessage().startsWith(err));
+ }
+ }
+
private void createRowFilteringPolicy(String policyName, String dbName, String tblName,
String user, String rowFilter) {
String json = String.format("{\n" +
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
index 82714d41e..88b49b1b1 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking_struct_in_select_list.test
@@ -16,6 +16,13 @@ select id, str, alltypes from functional_orc_def.complextypes_structs
INT,STRING,STRING
====
---- QUERY
+# A column mask on a top-level STRUCT column ('tiny_struct') is rejected too: the same
+# scalar-mask limitation applies to any complex type (IMPALA-15130).
+select id, tiny_struct from functional_orc_def.complextypes_structs
+---- CATCH
+Column masking is not supported for complex type column 'functional_orc_def.complextypes_structs.tiny_struct'
+====
+---- QUERY
select id, int_array from functional_orc_def.complextypestbl
---- RESULTS
NULL,'[1,2,3]'
@@ -30,20 +37,12 @@ NULL,'[-1]'
bigint,string
====
---- QUERY
-# There is a column mask added for int_array_array, but it is ignored.
-# It is not clear whether we should support this (RANGER-3525).
+# A column mask on the complex column 'int_array_array' cannot be applied: a scalar mask
+# transform cannot express a masked ARRAY value, so the query is rejected (IMPALA-15130)
+# rather than silently returning the column unmasked.
select id, int_array_array from functional_orc_def.complextypestbl
----- RESULTS
-NULL,'[[-1,-2],[]]'
-NULL,'[[1,2],[3,4]]'
-NULL,'[[null,1,2,null],[3,null,4],[],null]'
-NULL,'[null]'
-NULL,'[]'
-NULL,'NULL'
-NULL,'NULL'
-NULL,'[null,[5,6]]'
----- TYPES
-bigint,string
+---- CATCH
+Column masking is not supported for complex type column 'functional_orc_def.complextypestbl.int_array_array'
====
---- QUERY
select id, int_map from functional_orc_def.complextypestbl
@@ -60,18 +59,8 @@ NULL,'{"k1":null,"k3":null}'
bigint,string
====
---- QUERY
-# There is a column mask added for int_map_array, but it is ignored.
-# It is not clear whether we should support this (RANGER-3525).
+# Likewise, a column mask on the complex column 'int_map_array' is rejected.
select id, int_map_array from functional_orc_def.complextypestbl
----- RESULTS
-NULL,'[{},{"k1":1},{},{}]'
-NULL,'[{"k1":1}]'
-NULL,'[{"k3":null,"k1":1},null,{}]'
-NULL,'[null,null]'
-NULL,'[]'
-NULL,'NULL'
-NULL,'NULL'
-NULL,'NULL'
----- TYPES
-bigint,string
+---- CATCH
+Column masking is not supported for complex type column 'functional_orc_def.complextypestbl.int_map_array'
====
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index 1f1eecc9c..413183968 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -3175,20 +3175,19 @@ class TestRangerLocalCatalog(TestRanger):
self.run_test_case("QueryTest/ranger_column_masking", vector,
test_file_vars={'$UNIQUE_DB': unique_database})
# Add a policy on a primitive column of a table which contains nested columns.
+ # Masks on top-level complex-typed columns are rejected since IMPALA-15130; that is
+ # covered separately by AuthorizationStmtTest.testColumnMaskingOnComplexColumn().
for db in ['functional_parquet', 'functional_orc_def']:
TestRanger._add_column_masking_policy(
unique_name + str(policy_cnt), user, db, "complextypestbl",
"id", "CUSTOM", "100 * {col}")
policy_cnt += 1
- # Add policies on a nested column though they won't be recognized (same as Hive).
+ # A mask on a nested struct field is not recognized (same as Hive), so the column
+ # is returned unmasked. Keep it here to exercise that path end-to-end.
TestRanger._add_column_masking_policy(
unique_name + str(policy_cnt), user, db, "complextypestbl",
"nested_struct.a", "CUSTOM", "100 * {col}")
policy_cnt += 1
- TestRanger._add_column_masking_policy(
- unique_name + str(policy_cnt), user, db, "complextypestbl",
- "int_array", "MASK_NULL")
- policy_cnt += 1
self.execute_query_expect_success(admin_client, "refresh authorization")
self.run_test_case("QueryTest/ranger_column_masking_complex_types", vector,
use_db=db)
@@ -3932,7 +3931,9 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite):
admin_client = self.create_impala_client(user=ADMIN)
policy_cnt = 0
try:
- # Add a policy on a primitive column of a table which contains nested columns.
+ # Scalar-column masks (str, id) are applied normally. Masks on complex-typed
+ # columns (tiny_struct, int_array_array, int_map_array) are rejected now
+ # (IMPALA-15130); the .test file selects those columns and expects an error.
TestRanger._add_column_masking_policy(
unique_name + str(policy_cnt), user, "functional_orc_def",
"complextypes_structs", "str", "MASK_NULL")
@@ -3951,7 +3952,7 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite):
policy_cnt += 1
TestRanger._add_column_masking_policy(
unique_name + str(policy_cnt), user, "functional_orc_def",
- "complextypestbl", "int_array_map", "MASK_NULL")
+ "complextypestbl", "int_map_array", "MASK_NULL")
policy_cnt += 1
self.execute_query_expect_success(admin_client, "refresh authorization")
self.run_test_case("QueryTest/ranger_column_masking_struct_in_select_list", vector,