You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2020/02/05 04:31:51 UTC
[impala] branch master updated: IMPALA-9330: Fix column masking in
nested tables + enable column masking by default
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new a17bea9 IMPALA-9330: Fix column masking in nested tables + enable column masking by default
a17bea9 is described below
commit a17bea916f289599360796c45e8267a6338af6f8
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Fri Jan 31 11:58:07 2020 +0800
IMPALA-9330: Fix column masking in nested tables + enable column masking by default
Column masking policies on primitive columns of a table which contains
nested types (though they won't be masked) will cause query failures.
To be specifit, if tableA(id int, int_array array<int>) has a masking
policy on column "id", all queries on "tableA" will fail, e.g.
select id from tableA;
select t.id, a.item from tableA t, t.int_array a;
Column masking is implemented by wrapping the underlying table/view with
a table masking view. However, as we don't support nested types in
SelectList, the table masking view can't expose nested columns of the
masked table, which causes collection refs not being resolved correctly.
This patch fixes the issue by 2 steps:
1) Expose nested columns of the underlying table in the output Type of
the table masking view (see InlineViewRef#createTupleDescriptor()).
So nested Paths in the original query block can be resolved.
2) For such kind of Paths, resolved them again inside the table masking
view. So they can point to the underlying table as what they mean
(see Analyzer#resolvePathWithMasking()). TupleDescriptor of such kind
of table masking view won't be materialized since the view is simple
enough that its query plan is just a ScanNode of the underlying
table. The whole query plan can be stitched as if the table is not
masked.
Note that one day when we support nested columns in SelectList, we may
don't need these 2 hacks.
This patch also adds some TRACE level loggings to improve debuggability,
and enables column masking by default.
Test changes in TestRanger.test_column_masking:
- Add column masking policy on a table containing nested types.
- Add queries on the masked tables. Some queries are borrowed from
existing tests for nested types.
Tests:
- Run CORE tests.
Change-Id: I1cc5565c64c1a4a56445b8edde59b1168f387791
Reviewed-on: http://gerrit.cloudera.org:8080/15108
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/common/global-flags.cc | 3 +-
.../java/org/apache/impala/analysis/Analyzer.java | 79 ++++-
.../org/apache/impala/analysis/BaseTableRef.java | 1 -
.../apache/impala/analysis/CollectionTableRef.java | 24 +-
.../org/apache/impala/analysis/InlineViewRef.java | 29 +-
.../main/java/org/apache/impala/analysis/Path.java | 26 ++
.../org/apache/impala/analysis/SelectStmt.java | 2 +-
.../java/org/apache/impala/analysis/SlotRef.java | 2 +-
.../java/org/apache/impala/analysis/TableRef.java | 2 +
.../org/apache/impala/authorization/TableMask.java | 7 +-
.../ranger/RangerAuthorizationChecker.java | 8 +-
.../apache/impala/planner/SingleNodePlanner.java | 6 +
.../org/apache/impala/service/BackendConfig.java | 4 +
.../authorization/AuthorizationStmtTest.java | 2 +
.../queries/QueryTest/ranger_column_masking.test | 353 +++++++++++++++++++++
tests/authorization/test_ranger.py | 14 +
16 files changed, 538 insertions(+), 24 deletions(-)
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index e94f033..539ece2 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -312,7 +312,8 @@ DEFINE_bool_hidden(use_customized_user_groups_mapper_for_ranger, false,
"If true, use the customized user-to-groups mapper when performing authorization via"
" Ranger.");
-DEFINE_bool(enable_column_masking, false, "If true, enable the column masking feature.");
+DEFINE_bool(enable_column_masking, true,
+ "If false, disable the column masking feature. Defaults to be true.");
// ++========================++
// || Startup flag graveyard ||
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index 4a9b758..c842d64 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -682,6 +682,9 @@ public class Analyzer {
for (String alias: aliases) {
aliasMap_.put(alias, result);
}
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Register aliases {} to tuple {}", aliases, result.debugString());
+ }
tableRefMap_.put(result.getId(), ref);
return result;
}
@@ -704,6 +707,9 @@ public class Analyzer {
// Return the table if it is already resolved. This also avoids the table being
// masked again.
if (tableRef.isResolved()) return tableRef;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Resolving TableRef {}", ToSqlUtils.getPathSql(tableRef.getPath()));
+ }
// Try to find a matching local view.
if (tableRef.getPath().size() == 1) {
// Searches the hierarchy of analyzers bottom-up for a registered local view with
@@ -722,7 +728,7 @@ public class Analyzer {
List<String> rawPath = tableRef.getPath();
Path resolvedPath = null;
try {
- resolvedPath = resolvePath(tableRef.getPath(), PathType.TABLE_REF);
+ resolvedPath = resolvePathWithMasking(rawPath, PathType.TABLE_REF);
} catch (AnalysisException e) {
// Register privilege requests to prefer reporting an authorization error over
// an analysis error. We should not accidentally reveal the non-existence of a
@@ -766,11 +772,14 @@ public class Analyzer {
table instanceof FeDataSourceTable);
resolvedTableRef = new BaseTableRef(tableRef, resolvedPath);
}
+ // Only do table masking when authorization is enabled and the authorization
+ // factory supports column masking. If both of these are false, return the unmasked
+ // table ref.
if (!doTableMasking || !getAuthzFactory().getAuthorizationConfig().isEnabled()
|| !getAuthzFactory().supportsColumnMasking()) {
return resolvedTableRef;
}
-
+ // Performing table masking.
AuthorizationChecker authChecker = getAuthzFactory().newAuthorizationChecker(
getCatalog().getAuthPolicy());
TableMask tableMask = new TableMask(authChecker, table, user_);
@@ -900,6 +909,55 @@ public class Analyzer {
public boolean hasEmptySpjResultSet() { return hasEmptySpjResultSet_; }
/**
+ * Resolves 'rawPath' according to the given path type. Deal with paths that got
+ * resolved to nested columns of table masking views.
+ */
+ public Path resolvePathWithMasking(List<String> rawPath, PathType pathType)
+ throws AnalysisException, TableLoadingException {
+ Path resolvedPath = resolvePath(rawPath, pathType);
+ // Skip normal resolution cases that don't relate to nested types.
+ if (pathType == PathType.TABLE_REF) {
+ if (resolvedPath.destTable() != null || !resolvedPath.isRootedAtTuple()) {
+ return resolvedPath;
+ }
+ } else if (pathType == PathType.SLOT_REF) {
+ if (!resolvedPath.getMatchedTypes().get(0).isStructType()) {
+ return resolvedPath;
+ }
+ } else if (pathType == PathType.STAR) {
+ if (!resolvedPath.destType().isStructType() || !resolvedPath.isRootedAtTuple()) {
+ return resolvedPath;
+ }
+ }
+ // In this case, resolvedPath is resolved on a nested column. Check if it's resolved
+ // on a table masking view. The root TableRef(table/view) could be at a parent query
+ // block (correlated case, e.g. "t.int_array" in query
+ // "SELECT ... FROM tbl t, (SELECT * FROM t.int_array) a" roots at "tbl t" which is
+ // in the parent block), so we should find the parent block first then we can find
+ // the root TableRef.
+ TupleId rootTupleId = resolvedPath.getRootDesc().getId();
+ Analyzer parentAnalyzer = findAnalyzer(rootTupleId);
+ TableRef rootTblRef = parentAnalyzer.getTableRef(rootTupleId);
+ Preconditions.checkNotNull(rootTblRef);
+ if (!rootTblRef.isTableMaskingView()) return resolvedPath;
+ // resolvedPath is resolved on a nested column of a table masking view. The view
+ // won't produce results of nested columns. It just exposes the nested columns of the
+ // underlying BaseTableRef in the fields of its output type. (See more in
+ // InlineViewRef#createTupleDescriptor()). We need to resolve 'rawPath' inside the
+ // view as if the underlying table is not masked. So the resolved path can point to
+ // the real table and be used to create materialized slot in the TupleDescriptor of
+ // the real table.
+ InlineViewRef tableMaskingView = (InlineViewRef) rootTblRef;
+ Preconditions.checkState(
+ tableMaskingView.getUnMaskedTableRef() instanceof BaseTableRef);
+ // Resolve rawPath inside the table masking view to point to the real table.
+ Path maskedPath = tableMaskingView.inlineViewAnalyzer_.resolvePath(
+ rawPath, pathType);
+ maskedPath.setPathBeforeMasking(resolvedPath);
+ return maskedPath;
+ }
+
+ /**
* Resolves the given raw path according to the given path type, as follows:
* SLOT_REF and STAR: Resolves the path in the context of all registered tuple
* descriptors, considering qualified as well as unqualified matches.
@@ -942,7 +1000,10 @@ public class Analyzer {
// List of all candidate paths with different roots. Paths in this list are initially
// unresolved and may be illegal with respect to the pathType.
List<Path> candidates = getTupleDescPaths(rawPath);
-
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Candidates for {} {}: {}", pathType, ToSqlUtils.getPathSql(rawPath),
+ candidates);
+ }
LinkedList<String> errors = Lists.newLinkedList();
if (pathType == PathType.SLOT_REF || pathType == PathType.STAR) {
// Paths rooted at all of the unique registered tuple descriptors.
@@ -971,16 +1032,22 @@ public class Analyzer {
candidates.add(new Path(tbl, rawPath.subList(tblNameIdx + 1, rawPath.size())));
}
}
+ LOG.trace("Replace candidates with {}", candidates);
}
Path result = resolvePaths(rawPath, candidates, pathType, errors);
if (result == null && resolveInAncestors && hasAncestors()) {
+ LOG.trace("Resolve in ancestors");
result = getParentAnalyzer().resolvePath(rawPath, pathType, true);
}
if (result == null) {
Preconditions.checkState(!errors.isEmpty());
throw new AnalysisException(errors.getFirst());
}
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Resolved {} {} to {}", pathType, ToSqlUtils.getPathSql(rawPath),
+ result.debugString());
+ }
return result;
}
@@ -1033,7 +1100,10 @@ public class Analyzer {
List<Path> legalPaths = new ArrayList<>();
for (Path p: paths) {
- if (!p.resolve()) continue;
+ if (!p.resolve()) {
+ LOG.trace("Can't resolve path {}", p);
+ continue;
+ }
// Check legality of the resolved path.
if (p.isRootedAtTuple() && !isVisible(p.getRootDesc().getId())) {
@@ -1111,6 +1181,7 @@ public class Analyzer {
}
return null;
}
+ if (LOG.isTraceEnabled()) LOG.trace("Legal candidate: {}", legalPaths.get(0));
return legalPaths.get(0);
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java b/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
index 4c5fca9..64d2799 100644
--- a/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
@@ -84,7 +84,6 @@ public class BaseTableRef extends TableRef {
return getTable().getTableName().toSql() + aliasSql + tableSampleSql + tableHintsSql;
}
- public String debugString() { return tableRefToSql(); }
@Override
protected TableRef clone() { return new BaseTableRef(this); }
diff --git a/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
index 47564a6..9461280 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
@@ -86,16 +86,24 @@ public class CollectionTableRef extends TableRef {
collectionExpr_ = new SlotRef(parentSlotDesc);
// Must always be materialized to ensure the correct cardinality after unnesting.
analyzer.materializeSlots(collectionExpr_);
- Analyzer parentAnalyzer =
- analyzer.findAnalyzer(resolvedPath_.getRootDesc().getId());
+ TupleId rootTupleId = resolvedPath_.getRootDesc().getId();
+ if (resolvedPath_.isMaskedPath()) {
+ // Use tuple id of the table masking view so we can find it in current block or
+ // parent blocks. See how such kind of Paths are resolved in
+ // Analyzer#resolvePathWithMasking().
+ Preconditions.checkState(resolvedPath_.getPathBeforeMasking().isRootedAtTuple());
+ rootTupleId = resolvedPath_.getPathBeforeMasking().getRootDesc().getId();
+ }
+ Analyzer parentAnalyzer = analyzer.findAnalyzer(rootTupleId);
Preconditions.checkNotNull(parentAnalyzer);
- if (parentAnalyzer != analyzer) {
- TableRef parentRef =
- parentAnalyzer.getTableRef(resolvedPath_.getRootDesc().getId());
+ if (parentAnalyzer != analyzer) { // Correlated to a TableRef in a parent block.
+ TableRef parentRef = parentAnalyzer.getTableRef(rootTupleId);
Preconditions.checkNotNull(parentRef);
- // InlineViews are currently not supported as a parent ref.
- Preconditions.checkState(!(parentRef instanceof InlineViewRef));
- correlatedTupleIds_.add(parentRef.getId());
+ // InlineViews are currently not supported as a parent ref expects
+ // TableMaskingViews.
+ Preconditions.checkState(!(parentRef instanceof InlineViewRef)
+ || parentRef.isTableMaskingView());
+ correlatedTupleIds_.add(resolvedPath_.getRootDesc().getId());
}
}
if (!isRelative()) {
diff --git a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
index 43ed5ef..41106f0 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.impala.authorization.TableMask;
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.ColumnStats;
+import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.FeView;
import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
@@ -42,7 +43,7 @@ import com.google.common.collect.Sets;
* from a query string or represent a reference to a local or catalog view.
*/
public class InlineViewRef extends TableRef {
- private final static Logger LOG = LoggerFactory.getLogger(SelectStmt.class);
+ private final static Logger LOG = LoggerFactory.getLogger(InlineViewRef.class);
// Catalog or local view that is referenced.
// Null for inline views parsed directly from a query string.
@@ -154,10 +155,12 @@ public class InlineViewRef extends TableRef {
List<Column> columns = resolvedPath.getRootTable().getColumnsInHiveOrder();
List<SelectListItem> items = Lists.newArrayListWithCapacity(columns.size());
for (Column col: columns) {
+ if (col.getType().isComplexType()) continue;
// TODO: only add materialized columns to avoid introducing new privilege
// requirements (IMPALA-9223)
items.add(new SelectListItem(
- tableMask.createColumnMask(col.getName(), col.getType()), col.getName()));
+ tableMask.createColumnMask(col.getName(), col.getType()),
+ /*alias*/ col.getName()));
}
SelectList selectList = new SelectList(items);
FromClause fromClause = new FromClause(Lists.newArrayList(tableRef));
@@ -306,6 +309,28 @@ public class InlineViewRef extends TableRef {
}
fields.add(new StructField(colAlias, selectItemExpr.getType(), null));
}
+ // If this is a table masking view, the underlying table is wrapped by this so its
+ // nested columns are not visible to the original query block, because we can't
+ // expose nested columns in the SelectList. However, we can expose nested columns
+ // in this view's output type which is a StructType containing fields of all result
+ // columns. The output type is used to resolve Paths (see Path#resolve()).
+ // By exposing nested columns in the output type, Paths in the original query block
+ // can still be recognized and resolved.
+ if (isTableMaskingView_) {
+ TableRef tblRef = getUnMaskedTableRef();
+ if (tblRef instanceof BaseTableRef) {
+ BaseTableRef baseTbl = (BaseTableRef) tblRef;
+ FeTable tbl = baseTbl.resolvedPath_.getRootTable();
+ for (Column col : tbl.getColumnsInHiveOrder()) {
+ if (!col.getType().isComplexType()) continue;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Add {} (type={}) to output type of table mask view",
+ col.getName(), col.getType().toSql());
+ }
+ fields.add(new StructField(col.getName(), col.getType(), null));
+ }
+ }
+ }
// Create the non-materialized tuple and set its type.
TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor(
diff --git a/fe/src/main/java/org/apache/impala/analysis/Path.java b/fe/src/main/java/org/apache/impala/analysis/Path.java
index b3cd9e4..3049382 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Path.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Path.java
@@ -20,6 +20,7 @@ package org.apache.impala.analysis;
import java.util.ArrayList;
import java.util.List;
+import com.google.common.base.Objects;
import org.apache.impala.catalog.ArrayType;
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.FeTable;
@@ -31,6 +32,8 @@ import org.apache.impala.catalog.Type;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Represents a resolved or unresolved dot-separated path that is rooted at a registered
@@ -105,6 +108,8 @@ import com.google.common.collect.Lists;
* of the query block that it is used in.
*/
public class Path {
+ private final static Logger LOG = LoggerFactory.getLogger(Path.class);
+
// Implicit field names of collections.
public static final String ARRAY_ITEM_FIELD_NAME = "item";
public static final String ARRAY_POS_FIELD_NAME = "pos";
@@ -150,6 +155,9 @@ public class Path {
// Caches the result of getAbsolutePath() to avoid re-computing it.
private List<Integer> absolutePath_ = null;
+ // Resolved path before we resolved it again inside the table masking view.
+ private Path pathBeforeMasking_ = null;
+
/**
* Constructs a Path rooted at the given rootDesc.
*/
@@ -220,8 +228,12 @@ public class Path {
while (rawPathIdx < rawPath_.size()) {
if (!currentType.isComplexType()) return false;
StructType structType = getTypeAsStruct(currentType);
+ if (LOG.isTraceEnabled()) LOG.trace("structType: {}", structType.toSql());
// Resolve explicit path.
StructField field = structType.getField(rawPath_.get(rawPathIdx));
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("field: {}", field == null ? null : field.toSql(0));
+ }
if (field == null) {
// Resolve implicit path.
if (structType instanceof CollectionStructType) {
@@ -299,6 +311,12 @@ public class Path {
public boolean isRootedAtTuple() { return rootDesc_ != null; }
public List<String> getRawPath() { return rawPath_; }
public boolean isResolved() { return isResolved_; }
+ public boolean isMaskedPath() { return pathBeforeMasking_ != null; }
+ public Path getPathBeforeMasking() { return pathBeforeMasking_; }
+ public void setPathBeforeMasking(Path p) {
+ Preconditions.checkState(p.isResolved());
+ pathBeforeMasking_ = p;
+ }
public List<Type> getMatchedTypes() {
Preconditions.checkState(isResolved_);
@@ -449,6 +467,14 @@ public class Path {
return pathRoot + "." + Joiner.on(".").join(rawPath_);
}
+ public String debugString() {
+ return Objects.toStringHelper(this)
+ .add("rootTable", rootTable_)
+ .add("rootDesc", rootDesc_)
+ .add("rawPath", rawPath_)
+ .toString();
+ }
+
/**
* Returns a raw path from a known root alias and field name.
*/
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
index 5b747df..456ac0a 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectStmt.java
@@ -441,7 +441,7 @@ public class SelectStmt extends QueryStmt {
throws AnalysisException {
Path resolvedPath = null;
try {
- resolvedPath = analyzer.resolvePath(rawPath, PathType.STAR);
+ resolvedPath = analyzer.resolvePathWithMasking(rawPath, PathType.STAR);
} catch (TableLoadingException e) {
// Should never happen because we only check registered table aliases.
Preconditions.checkState(false);
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
index 7bd3b4d..0fe4d8b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
@@ -94,7 +94,7 @@ public class SlotRef extends Expr {
Preconditions.checkState(rawPath_ != null);
Path resolvedPath = null;
try {
- resolvedPath = analyzer.resolvePath(rawPath_, PathType.SLOT_REF);
+ resolvedPath = analyzer.resolvePathWithMasking(rawPath_, PathType.SLOT_REF);
} catch (TableLoadingException e) {
// Should never happen because we only check registered table aliases.
Preconditions.checkState(false);
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableRef.java b/fe/src/main/java/org/apache/impala/analysis/TableRef.java
index cd5f51c..47be874 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableRef.java
@@ -576,6 +576,8 @@ public class TableRef extends StmtNode {
if (onClause_ != null) onClause_ = rewriter.rewrite(onClause_, analyzer);
}
+ public String debugString() { return tableRefToSql(); }
+
protected String tableRefToSql() { return tableRefToSql(DEFAULT); }
protected String tableRefToSql(ToSqlOptions options) {
diff --git a/fe/src/main/java/org/apache/impala/authorization/TableMask.java b/fe/src/main/java/org/apache/impala/authorization/TableMask.java
index 9323b96..b210568 100644
--- a/fe/src/main/java/org/apache/impala/authorization/TableMask.java
+++ b/fe/src/main/java/org/apache/impala/authorization/TableMask.java
@@ -17,6 +17,8 @@
package org.apache.impala.authorization;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.Parser;
import org.apache.impala.analysis.SelectStmt;
@@ -66,13 +68,16 @@ public class TableMask {
*/
public Expr createColumnMask(String colName, Type colType)
throws InternalException, AnalysisException {
+ Preconditions.checkState(!colType.isComplexType());
String maskedValue = authChecker_.createColumnMask(user_, dbName_, tableName_,
colName);
if (LOG.isTraceEnabled()) {
LOG.trace("Performing column masking on table {}.{}: {} => {}",
dbName_, tableName_, colName, maskedValue);
}
- if (maskedValue == null) return new SlotRef(colName);
+ if (maskedValue == null || maskedValue.equals(colName)) { // Don't need masking.
+ return new SlotRef(Lists.newArrayList(colName));
+ }
SelectStmt maskStmt = (SelectStmt) Parser.parse(
String.format("SELECT CAST(%s AS %s)", maskedValue, colType));
if (maskStmt.getSelectList().getItems().size() != 1 || maskStmt.hasGroupByClause()
diff --git a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationChecker.java b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationChecker.java
index 264da9c..fe4b77c 100644
--- a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationChecker.java
+++ b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationChecker.java
@@ -72,7 +72,6 @@ public class RangerAuthorizationChecker extends BaseAuthorizationChecker {
public static final String SELECT_ACCESS_TYPE = "select";
private final RangerImpalaPlugin plugin_;
- private boolean isColumnMaskingEnabled_;
public RangerAuthorizationChecker(AuthorizationConfig authzConfig) {
super(authzConfig);
@@ -81,7 +80,6 @@ public class RangerAuthorizationChecker extends BaseAuthorizationChecker {
plugin_ = new RangerImpalaPlugin(
rangerConfig.getServiceType(), rangerConfig.getAppId());
plugin_.init();
- isColumnMaskingEnabled_ = BackendConfig.INSTANCE.isColumnMaskingEnabled();
}
@Override
@@ -189,8 +187,9 @@ public class RangerAuthorizationChecker extends BaseAuthorizationChecker {
protected void authorizeRowFilterAndColumnMask(User user,
List<PrivilegeRequest> privilegeRequests)
throws AuthorizationException, InternalException {
+ boolean isColumnMaskingEnabled = BackendConfig.INSTANCE.isColumnMaskingEnabled();
for (PrivilegeRequest request : privilegeRequests) {
- if (!isColumnMaskingEnabled_
+ if (!isColumnMaskingEnabled
&& request.getAuthorizable().getType() == Type.COLUMN) {
authorizeColumnMask(user,
request.getAuthorizable().getDbName(),
@@ -278,8 +277,7 @@ public class RangerAuthorizationChecker extends BaseAuthorizationChecker {
*/
private void authorizeColumnMask(User user, String dbName, String tableName,
String columnName) throws InternalException, AuthorizationException {
- if (!isColumnMaskingEnabled_
- && evalColumnMask(user, dbName, tableName, columnName).isMaskEnabled()) {
+ if (evalColumnMask(user, dbName, tableName, columnName).isMaskEnabled()) {
throw new AuthorizationException(String.format(
"Column masking is disabled by --enable_column_masking flag. Can't access " +
"column %s.%s.%s that has column masking policy.",
diff --git a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
index 9dff663..01921f2 100644
--- a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
@@ -823,6 +823,12 @@ public class SingleNodePlanner {
parentRefs.get(i).setLeftTblRef(parentRefs.get(i - 1));
}
for (SubplanRef subplanRef: subplanRefs) subplanRef.tblRef.setLeftTblRef(null);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("parentRefs: {}",
+ parentRefs.stream().map(TableRef::debugString).reduce(", ", String::concat));
+ LOG.trace("subplanRefs: {}", subplanRefs.stream().map(r -> r.tblRef.debugString())
+ .reduce(", ", String::concat));
+ }
}
/**
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index 45465c2..ac7f66b 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -205,6 +205,10 @@ public class BackendConfig {
return backendCfg_.use_customized_user_groups_mapper_for_ranger;
}
+ public void setColumnMaskingEnabled(boolean columnMaskingEnabled) {
+ backendCfg_.setEnable_column_masking(columnMaskingEnabled);
+ }
+
public boolean isColumnMaskingEnabled() { return backendCfg_.enable_column_masking; }
// Inits the auth_to_local configuration in the static KerberosName class.
diff --git a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
index 06ed49b..de4510e 100644
--- a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
@@ -2865,6 +2865,7 @@ public class AuthorizationStmtTest extends AuthorizationTestBase {
String policyName = "col_mask";
for (String tableName: new String[]{"alltypes", "alltypes_view"}) {
+ BackendConfig.INSTANCE.setColumnMaskingEnabled(false);
String json = String.format("{\n" +
" \"name\": \"%s\",\n" +
" \"policyType\": 1,\n" +
@@ -2978,6 +2979,7 @@ public class AuthorizationStmtTest extends AuthorizationTestBase {
onServer(TPrivilegeLevel.ALL));
} finally {
deleteRangerPolicy(policyName);
+ BackendConfig.INSTANCE.setColumnMaskingEnabled(true);
}
}
}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking.test
index 4c71a80..02451b3 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_column_masking.test
@@ -398,3 +398,356 @@ show create view $UNIQUE_DB.masked_view;
---- RESULTS
'CREATE VIEW $UNIQUE_DB.masked_view AS\nSELECT id FROM functional.alltypestiny'
====
+---- QUERY
+# Test queries on complex types table.
+select id from functional_parquet.complextypestbl
+---- RESULTS
+100
+200
+300
+400
+500
+600
+700
+800
+---- TYPES
+BIGINT
+====
+---- QUERY
+# Test queries on complex types table.
+select * from functional_parquet.complextypestbl
+---- RESULTS
+100
+200
+300
+400
+500
+600
+700
+800
+---- TYPES
+BIGINT
+====
+---- QUERY
+# Test resolving nested column of the masked table.
+select id, nested_struct.a from functional_parquet.complextypestbl
+---- RESULTS
+100,1
+200,NULL
+300,NULL
+400,NULL
+500,NULL
+600,NULL
+700,7
+800,-1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# Test resolving nested columns in expending star expression.
+select id, nested_struct.* from functional_parquet.complextypestbl
+---- RESULTS
+100,1
+200,NULL
+300,NULL
+400,NULL
+500,NULL
+600,NULL
+700,7
+800,-1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# Test resolving nested column in function.
+select count(id), count(nested_struct.a) from functional_parquet.complextypestbl
+---- RESULTS
+8,3
+---- TYPES
+BIGINT,BIGINT
+====
+---- QUERY
+# Test predicates on masked columns and nested columns.
+select id, nested_struct.a from functional_parquet.complextypestbl t
+where id = 100 and nested_struct.a = 1;
+---- RESULTS
+100,1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# Test resolving nested collection of the nested table. Should resolve 't.int_array'
+# correctly though 'complextypestbl' will be masked into a table masking view.
+select pos, item from functional_parquet.complextypestbl t, t.int_array
+---- RESULTS
+0,-1
+0,1
+0,NULL
+1,1
+1,2
+2,2
+2,3
+3,NULL
+4,3
+5,NULL
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# Regression test when 'complextypestbl' is not used as TableRef.
+select pos, item from functional_parquet.complextypestbl.int_array
+---- RESULTS
+0,-1
+0,1
+0,NULL
+1,1
+1,2
+2,2
+2,3
+3,NULL
+4,3
+5,NULL
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# Test resolving nested columns and nested collections.
+select id, nested_struct.a, a.pos, a.item
+from functional_parquet.complextypestbl t, t.int_array a
+---- RESULTS
+100,1,0,1
+100,1,1,2
+100,1,2,3
+200,NULL,0,NULL
+200,NULL,1,1
+200,NULL,2,2
+200,NULL,3,NULL
+200,NULL,4,3
+200,NULL,5,NULL
+800,-1,0,-1
+---- TYPES
+BIGINT,INT,BIGINT,INT
+====
+---- QUERY
+# Test different JOINs comparing to the above test.
+select id, nested_struct.a, a.pos, a.item
+from functional_parquet.complextypestbl t join t.int_array a
+---- RESULTS
+100,1,0,1
+100,1,1,2
+100,1,2,3
+200,NULL,0,NULL
+200,NULL,1,1
+200,NULL,2,2
+200,NULL,3,NULL
+200,NULL,4,3
+200,NULL,5,NULL
+800,-1,0,-1
+---- TYPES
+BIGINT,INT,BIGINT,INT
+====
+---- QUERY
+# Test different JOINs.
+select id, nested_struct.a, a.pos, a.item
+from functional_parquet.complextypestbl t left join t.int_array a
+---- RESULTS
+100,1,0,1
+100,1,1,2
+100,1,2,3
+200,NULL,0,NULL
+200,NULL,1,1
+200,NULL,2,2
+200,NULL,3,NULL
+200,NULL,4,3
+200,NULL,5,NULL
+300,NULL,NULL,NULL
+400,NULL,NULL,NULL
+500,NULL,NULL,NULL
+600,NULL,NULL,NULL
+700,7,NULL,NULL
+800,-1,0,-1
+---- TYPES
+BIGINT,INT,BIGINT,INT
+====
+---- QUERY
+# Test different JOINs.
+select id, nested_struct.a, a.pos, a.item
+from functional_parquet.complextypestbl t right join t.int_array a
+---- RESULTS
+100,1,0,1
+100,1,1,2
+100,1,2,3
+200,NULL,0,NULL
+200,NULL,1,1
+200,NULL,2,2
+200,NULL,3,NULL
+200,NULL,4,3
+200,NULL,5,NULL
+800,-1,0,-1
+---- TYPES
+BIGINT,INT,BIGINT,INT
+====
+---- QUERY
+# Test different JOINs.
+select id, nested_struct.a, a.pos, a.item
+from functional_parquet.complextypestbl t full outer join t.int_array a
+---- RESULTS
+100,1,0,1
+100,1,1,2
+100,1,2,3
+200,NULL,0,NULL
+200,NULL,1,1
+200,NULL,2,2
+200,NULL,3,NULL
+200,NULL,4,3
+200,NULL,5,NULL
+300,NULL,NULL,NULL
+400,NULL,NULL,NULL
+500,NULL,NULL,NULL
+600,NULL,NULL,NULL
+700,7,NULL,NULL
+800,-1,0,-1
+---- TYPES
+BIGINT,INT,BIGINT,INT
+====
+---- QUERY
+# Test function and predicates on nested columns of the masked table.
+select count(nested_struct.a) from functional_parquet.complextypestbl t, t.int_array a
+where id = 100 and nested_struct.a = 1
+---- RESULTS
+3
+---- TYPES
+BIGINT
+====
+---- QUERY
+# Test on a deeper nested collection 'int_array_array'.
+select id, nested_struct.a, aa.item
+from functional_parquet.complextypestbl t, t.int_array_array.item aa
+---- RESULTS
+100,1,1
+100,1,3
+100,1,2
+100,1,4
+200,NULL,NULL
+200,NULL,3
+200,NULL,1
+200,NULL,NULL
+200,NULL,2
+200,NULL,4
+200,NULL,NULL
+700,7,5
+700,7,6
+800,-1,-1
+800,-1,-2
+---- TYPES
+BIGINT,INT,INT
+====
+---- QUERY
+# Test on several nested collections.
+select id, nested_struct.a as field, a.item, aa.item
+from functional_parquet.complextypestbl t, t.int_array a, t.int_array_array.item aa
+where nested_struct.a = -1
+---- RESULTS
+800,-1,-1,-1
+800,-1,-1,-2
+---- TYPES
+BIGINT,INT,INT,INT
+====
+---- QUERY
+# Test on map type.
+select id, key, value from functional_parquet.complextypestbl t, t.int_map
+---- RESULTS
+100,'k1',1
+100,'k2',100
+200,'k1',2
+200,'k2',NULL
+700,'k1',NULL
+700,'k3',NULL
+800,'k1',-1
+---- TYPES
+BIGINT,STRING,INT
+====
+---- QUERY
+# Test on deep nested column 'nested_struct.b'.
+select id, item from functional_parquet.complextypestbl t, t.nested_struct.b
+---- RESULTS
+100,1
+200,NULL
+700,2
+700,3
+700,NULL
+800,-1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# Test on correlated CollectionTableRefs. This query is copied from nested-types-scanner-multiple-materialization.test.
+select id, item from functional_parquet.complextypestbl t,
+(select item from t.int_array where item = 2
+ union all
+ select item from t.int_array where item != 2
+ union all
+ select item from t.int_array where item is null) v
+---- RESULTS
+100,1
+100,2
+100,3
+200,1
+200,2
+200,3
+200,NULL
+200,NULL
+200,NULL
+800,-1
+---- TYPES
+BIGINT,INT
+====
+---- QUERY
+# Test on correlated CollectionTableRefs. This query is copied from nested-types-scanner-multiple-materialization.test.
+select id, e, f from functional_parquet.complextypestbl t,
+(select e, f from t.nested_struct.c.d.item where e = 10
+ union all
+ select e, f from t.nested_struct.c.d.item where e != 10
+ union all
+ select e, f from t.nested_struct.c.d.item where e is null) v
+---- RESULTS
+100,-10,'bbb'
+100,10,'aaa'
+100,11,'c'
+200,-10,'bbb'
+200,10,'aaa'
+200,11,'c'
+200,NULL,'NULL'
+200,NULL,'NULL'
+200,NULL,'NULL'
+200,NULL,'NULL'
+700,NULL,'NULL'
+800,-1,'nonnullable'
+---- TYPES
+BIGINT,INT,STRING
+====
+---- QUERY
+# Test on relative CollectionTableRefs. This query is copied from nested-types-scanner-multiple-materialization.test.
+select id, int_array.item, a2.item, a3.item,
+nested_struct.a, b.item, d2.e, d2.f, d3.e, d3.f
+from functional_parquet.complextypestbl t,
+t.int_array,
+t.int_array_array a1, a1.item a2,
+t.int_array_array.item a3,
+t.nested_struct.b,
+t.nested_struct.c.d, d.item d2,
+t.nested_struct.c.d.item d3
+where a2.item = 1 and a3.item = 2 and d2.e = 10 and d3.e = -10
+---- RESULTS
+100,1,1,2,1,1,10,'aaa',-10,'bbb'
+100,2,1,2,1,1,10,'aaa',-10,'bbb'
+100,3,1,2,1,1,10,'aaa',-10,'bbb'
+200,1,1,2,NULL,NULL,10,'aaa',-10,'bbb'
+200,2,1,2,NULL,NULL,10,'aaa',-10,'bbb'
+200,3,1,2,NULL,NULL,10,'aaa',-10,'bbb'
+200,NULL,1,2,NULL,NULL,10,'aaa',-10,'bbb'
+200,NULL,1,2,NULL,NULL,10,'aaa',-10,'bbb'
+200,NULL,1,2,NULL,NULL,10,'aaa',-10,'bbb'
+---- TYPES
+BIGINT,INT,INT,INT,INT,INT,INT,STRING,INT,STRING
+====
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index 2e0d1fa..3d2a7ea 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -799,6 +799,20 @@ class TestRanger(CustomClusterTestSuite):
unique_name + str(policy_cnt), user, "functional", "alltypes", "string_col",
"CUSTOM", "concat({col}, 'ttt')")
policy_cnt += 1
+ # Add a policy on a primitive column of a table which contains nested columns.
+ TestRanger._add_column_masking_policy(
+ unique_name + str(policy_cnt), user, "functional_parquet", "complextypestbl",
+ "id", "CUSTOM", "100 * {col}")
+ policy_cnt += 1
+ # Add policies on a nested column though they won't be recognized (same as Hive).
+ TestRanger._add_column_masking_policy(
+ unique_name + str(policy_cnt), user, "functional_parquet", "complextypestbl",
+ "nested_struct.a", "CUSTOM", "100 * {col}")
+ policy_cnt += 1
+ TestRanger._add_column_masking_policy(
+ unique_name + str(policy_cnt), user, "functional_parquet", "complextypestbl",
+ "int_array", "MASK_NULL")
+ policy_cnt += 1
self.execute_query_expect_success(admin_client, "refresh authorization",
user=ADMIN)
self.run_test_case("QueryTest/ranger_column_masking", vector,