You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2021/06/11 07:08:40 UTC

[impala] 02/03: IMPALA-10728: Check access privileges inside masking expressions

This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch branch-4.0.0
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 7d593f1aba3ac5a01510a128bdf55decfc52aba6
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Mon Jun 7 15:59:25 2021 +0800

    IMPALA-10728: Check access privileges inside masking expressions
    
    Row-filtering/column-masking policies may introduce new expressions or
    tables to the original query, e.g. a column-masking policy can reference
    other columns that are not used by the original query, a row-filtering
    policy can filter rows based on a subquery on other tables.
    
    Previously, we intend to not checking accesses on these expressions and
    tables. One reason is that they are hidden and users are not aware of
    them. The other reason is the expressions are considered to be evaluated
    by the admin (who creates these policies and has access to all
    resources). The same for masking policies, i.e. if the hidden tables
    have masking policies, those policies are not applied recursively. This
    also avoids infinitely recursively masking.
    
    Hive has the same behavior on masking policies. However, Hive does check
    access privileges inside the masking expressions. To avoid breaking
    users who may have already depended on this, we adjust our behavior to
    check access privileges inside the masking expressions as well. Due to
    this, audits will also be generated on accessing the additional columns.
    
    Implementation:
    Before this patch, we only collect privilege requests when analyzing the
    original query. When masking expressions are applied and query rewrite
    rules are applied, we re-analyze the AST without collecting privilege
    requests. This patch adjusts the behavior to also collect them in the
    re-analyzing pass after masking expressions are applied.
    
    Tests
     - Add FE tests.
     - Adjust existing FE audit test to verify the additional audits.
     - Run CORE tests.
    
    Change-Id: I772fa2f5d189b1272e8419ed1fb5b34d373d3601
    Reviewed-on: http://gerrit.cloudera.org:8080/17550
    Reviewed-by: Aman Sinha <am...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 .../apache/impala/analysis/AnalysisContext.java    | 51 +++++++++---------
 .../authorization/AuthorizationStmtTest.java       | 62 ++++++++++++++++++++++
 .../authorization/ranger/RangerAuditLogTest.java   | 26 +++++----
 .../org/apache/impala/common/FrontendTestBase.java |  4 +-
 4 files changed, 107 insertions(+), 36 deletions(-)

diff --git a/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java b/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
index 48bed79..de55559 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AnalysisContext.java
@@ -501,37 +501,39 @@ public class AnalysisContext {
         Lists.newArrayList(analysisResult_.stmt_.getColLabels());
 
     // Apply column/row masking, expr, setop, and subquery rewrites.
-    boolean reAnalyze = false;
+    boolean shouldReAnalyze = false;
     if (authzFactory_.getAuthorizationConfig().isEnabled()) {
-      reAnalyze = analysisResult_.stmt_.resolveTableMask(analysisResult_.analyzer_);
+      shouldReAnalyze = analysisResult_.stmt_.resolveTableMask(analysisResult_.analyzer_);
       // If any catalog table/view is replaced by table masking views, we need to
       // resolve them. Also re-analyze the SlotRefs to reference the output exprs of
       // the table masking views.
-      if (reAnalyze) {
-        reAnalyzeWithoutPrivChecks(stmtTableCache, authzCtx, origResultTypes,
-            origColLabels);
+      if (shouldReAnalyze) {
+        // To be consistent with Hive, privilege requests introduced by the masking views
+        // should also be collected (IMPALA-10728).
+        reAnalyze(stmtTableCache, authzCtx, origResultTypes, origColLabels,
+            /*collectPrivileges*/ true);
       }
-      reAnalyze = false;
+      shouldReAnalyze = false;
     }
     ExprRewriter rewriter = analysisResult_.analyzer_.getExprRewriter();
     if (analysisResult_.requiresExprRewrite()) {
       rewriter.reset();
       analysisResult_.stmt_.rewriteExprs(rewriter);
-      reAnalyze = rewriter.changed();
+      shouldReAnalyze = rewriter.changed();
     }
     if (analysisResult_.requiresSubqueryRewrite()) {
       new StmtRewriter.SubqueryRewriter().rewrite(analysisResult_);
-      reAnalyze = true;
+      shouldReAnalyze = true;
     }
     if (analysisResult_.requiresSetOperationRewrite()) {
       new StmtRewriter().rewrite(analysisResult_);
-      reAnalyze = true;
+      shouldReAnalyze = true;
     }
     if (analysisResult_.requiresAcidComplexScanRewrite()) {
       new StmtRewriter.AcidRewriter().rewrite(analysisResult_);
-      reAnalyze = true;
+      shouldReAnalyze = true;
     }
-    if (!reAnalyze) return;
+    if (!shouldReAnalyze) return;
 
     // For SetOperationStmt we must replace the query statement with the rewritten version
     // before re-analysis.
@@ -544,13 +546,14 @@ public class AnalysisContext {
       }
     }
 
-    reAnalyzeWithoutPrivChecks(stmtTableCache, authzCtx, origResultTypes, origColLabels);
+    reAnalyze(stmtTableCache, authzCtx, origResultTypes, origColLabels,
+        /*collectPrivileges*/ false);
     Preconditions.checkState(!analysisResult_.requiresSubqueryRewrite());
   }
 
-  private void reAnalyzeWithoutPrivChecks(StmtTableCache stmtTableCache,
-      AuthorizationContext authzCtx, List<Type> origResultTypes,
-      List<String> origColLabels) throws AnalysisException {
+  private void reAnalyze(StmtTableCache stmtTableCache, AuthorizationContext authzCtx,
+      List<Type> origResultTypes, List<String> origColLabels, boolean collectPrivileges)
+      throws AnalysisException {
     boolean isExplain = analysisResult_.isExplainStmt();
     // Some expressions, such as function calls with constant arguments, can get
     // folded into literals. Since literals do not require privilege requests, we
@@ -561,17 +564,20 @@ public class AnalysisContext {
 
     // Re-analyze the stmt with a new analyzer.
     analysisResult_.analyzer_ = createAnalyzer(stmtTableCache, authzCtx);
-    // We restore the privileges collected in the first pass below. So, no point in
-    // collecting them again.
-    analysisResult_.analyzer_.setEnablePrivChecks(false);
+    // Restore privilege requests found during the previous pass
+    for (PrivilegeRequest req : origPrivReqs) {
+      analysisResult_.analyzer_.registerPrivReq(req);
+    }
+    // Only collect privilege requests in need.
+    analysisResult_.analyzer_.setEnablePrivChecks(collectPrivileges);
     analysisResult_.stmt_.reset();
     try {
       analysisResult_.stmt_.analyze(analysisResult_.analyzer_);
-      analysisResult_.analyzer_.setEnablePrivChecks(true); // restore
+      analysisResult_.analyzer_.setEnablePrivChecks(true); // Always restore
     } catch (AnalysisException e) {
       LOG.error(String.format("Error analyzing the rewritten query.\n" +
           "Original SQL: %s\nRewritten SQL: %s", analysisResult_.stmt_.toSql(),
-          analysisResult_.stmt_.toSql(REWRITTEN)));
+          analysisResult_.stmt_.toSql(REWRITTEN)), e);
       throw e;
     }
     // Restore the original result types and column labels.
@@ -580,11 +586,6 @@ public class AnalysisContext {
     if (LOG.isTraceEnabled()) {
       LOG.trace("Rewritten SQL: " + analysisResult_.stmt_.toSql(REWRITTEN));
     }
-
-    // Restore privilege requests found during the first pass
-    for (PrivilegeRequest req : origPrivReqs) {
-      analysisResult_.analyzer_.registerPrivReq(req);
-    }
     if (isExplain) analysisResult_.stmt_.setIsExplain();
   }
 
diff --git a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
index ce2b6be..9b1f31d 100644
--- a/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationStmtTest.java
@@ -3212,6 +3212,68 @@ public class AuthorizationStmtTest extends AuthorizationTestBase {
   }
 
   /**
+   * Validates access privileges are checked inside column-masking/row-filtering
+   * expressions (IMPALA-10728).
+   */
+  @Test
+  public void testPrivInMaskingExprs() throws Exception {
+    // Create a row-filter that contains a subquery on another table. Verify that accesses
+    // inside the subquery are also checked.
+    try {
+      createRowFilteringPolicy("alltypes_subquery_filter", "functional", "alltypes",
+          user_.getShortName(), "id in (select id from functional.alltypestiny)");
+      String[] queries = {
+          "select * from functional.alltypes",
+          "select id from functional.alltypes",
+          "select int_col from functional.alltypes"
+      };
+      for (String q : queries) {
+        authorize(q)
+            .ok(onTable("functional", "alltypes", TPrivilegeLevel.SELECT),
+                onTable("functional", "alltypestiny", TPrivilegeLevel.SELECT))
+            .ok(onTable("functional", "alltypes", TPrivilegeLevel.SELECT),
+                onColumn("functional", "alltypestiny", "id", TPrivilegeLevel.SELECT))
+            // Can't read the table if missing SELECT privilege on column "id" of table
+            // "functional.alltypestiny".
+            .error(selectError("functional.alltypestiny"),
+                onTable("functional", "alltypes", TPrivilegeLevel.SELECT))
+            .error(selectError("functional.alltypestiny"),
+                onTable("functional", "alltypes", TPrivilegeLevel.SELECT),
+                onColumn("functional", "alltypestiny", "int_col",
+                    TPrivilegeLevel.SELECT));
+      }
+    } finally {
+      deleteRangerPolicy("alltypes_subquery_filter");
+    }
+    // Create a column-masking policy that masks a column using another column. Verify
+    // that accesses on both columns are checked. Create a column-masking policy that
+    // masks a column to NULL. Verify that access on the column are still checked.
+    try {
+      createColumnMaskingPolicy("alltypes_replace_id", "functional", "alltypes", "id",
+          user_.getShortName(), "CUSTOM", "int_col");
+      createColumnMaskingPolicy("alltypes_nullify_str", "functional", "alltypes",
+          "string_col", user_.getShortName(), "MASK_NULL", /*maskExpr*/null);
+      authorize("select id from functional.alltypes")
+          .ok(onTable("functional", "alltypes", TPrivilegeLevel.SELECT))
+          .ok(onColumn("functional", "alltypes", "id", TPrivilegeLevel.SELECT),
+              onColumn("functional", "alltypes", "int_col", TPrivilegeLevel.SELECT))
+          .error(selectError("functional.alltypes"),
+              onColumn("functional", "alltypes", "id", TPrivilegeLevel.SELECT))
+          .error(selectError("functional.alltypes"),
+              onColumn("functional", "alltypes", "int_col", TPrivilegeLevel.SELECT));
+      // Although string_col is masked as NULL, we still require access privilege on it.
+      authorize("select bool_col, string_col from functional.alltypes")
+          .ok(onColumn("functional", "alltypes", "bool_col", TPrivilegeLevel.SELECT),
+              onColumn("functional", "alltypes", "string_col", TPrivilegeLevel.SELECT))
+          .error(selectError("functional.alltypes"),
+              onColumn("functional", "alltypes", "bool_col", TPrivilegeLevel.SELECT));
+    } finally {
+      deleteRangerPolicy("alltypes_replace_id");
+      deleteRangerPolicy("alltypes_nullify_str");
+    }
+  }
+
+  /**
    * Validates Ranger's object ownership privileges. Note that we no longer have to add a
    * policy to the Ranger server to explicitly grant a user the access privileges of the
    * resources if the user is the owner of the resources.
diff --git a/fe/src/test/java/org/apache/impala/authorization/ranger/RangerAuditLogTest.java b/fe/src/test/java/org/apache/impala/authorization/ranger/RangerAuditLogTest.java
index e2d8065..2e75393 100644
--- a/fe/src/test/java/org/apache/impala/authorization/ranger/RangerAuditLogTest.java
+++ b/fe/src/test/java/org/apache/impala/authorization/ranger/RangerAuditLogTest.java
@@ -581,43 +581,49 @@ public class RangerAuditLogTest extends AuthorizationTestBase {
         assertNotEquals("Illegal policy id", -1, policyIds[i]);
       }
 
-      // Verify row filter audits. Note that columns used in the row filter won't create
+      // Verify row filter audits. Note that columns used in the row filter also creates
       // column access audits.
       authzOk(events -> {
-        assertEquals(3, events.size());
+        assertEquals(4, events.size());
         assertEquals("select bool_col from functional.alltypestiny",
             events.get(0).getRequestData());
         assertEventEquals("@table", "select", "functional/alltypestiny", 1,
             events.get(0));
         assertEventEquals("@column", "select", "functional/alltypestiny/bool_col", 1,
             events.get(1));
-        assertEventEquals("@table", "row_filter", "functional/alltypestiny", 1,
+        assertEventEquals("@column", "select", "functional/alltypestiny/id", 1,
             events.get(2));
-        assertEquals(events.get(2).getPolicyId(), policyIds[0]);
+        assertEventEquals("@table", "row_filter", "functional/alltypestiny", 1,
+            events.get(3));
+        assertEquals(events.get(3).getPolicyId(), policyIds[0]);
       }, "select bool_col from functional.alltypestiny",
           onTable("functional", "alltypestiny", TPrivilegeLevel.SELECT));
 
       authzOk(events -> {
-        assertEquals(2, events.size());
+        assertEquals(3, events.size());
         assertEquals("select 1 from functional.alltypestiny",
             events.get(0).getRequestData());
         assertEventEquals("@table", "select", "functional/alltypestiny", 1,
             events.get(0));
-        assertEventEquals("@table", "row_filter", "functional/alltypestiny", 1,
+        assertEventEquals("@column", "select", "functional/alltypestiny/id", 1,
             events.get(1));
-        assertEquals(events.get(1).getPolicyId(), policyIds[0]);
+        assertEventEquals("@table", "row_filter", "functional/alltypestiny", 1,
+            events.get(2));
+        assertEquals(events.get(2).getPolicyId(), policyIds[0]);
       }, "select 1 from functional.alltypestiny",
           onTable("functional", "alltypestiny", TPrivilegeLevel.SELECT));
 
       authzOk(events -> {
-        assertEquals(2, events.size());
+        assertEquals(3, events.size());
         assertEquals("select count(*) from functional.alltypestiny",
             events.get(0).getRequestData());
         assertEventEquals("@table", "select", "functional/alltypestiny", 1,
             events.get(0));
-        assertEventEquals("@table", "row_filter", "functional/alltypestiny", 1,
+        assertEventEquals("@column", "select", "functional/alltypestiny/id", 1,
             events.get(1));
-        assertEquals(events.get(1).getPolicyId(), policyIds[0]);
+        assertEventEquals("@table", "row_filter", "functional/alltypestiny", 1,
+            events.get(2));
+        assertEquals(events.get(2).getPolicyId(), policyIds[0]);
       }, "select count(*) from functional.alltypestiny",
           onTable("functional", "alltypestiny", TPrivilegeLevel.SELECT));
 
diff --git a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
index d32baea..bdf0815 100644
--- a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
+++ b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
@@ -61,6 +61,7 @@ import org.apache.impala.thrift.TAccessEvent;
 import org.apache.impala.thrift.TQueryOptions;
 import org.apache.impala.thrift.TSessionState;
 import org.apache.impala.util.EventSequence;
+import org.apache.impala.util.TSessionStateUtil;
 import org.junit.Assert;
 
 import com.google.common.base.Preconditions;
@@ -315,8 +316,9 @@ public class FrontendTestBase extends AbstractFrontendTest {
     try (FrontendProfile.Scope scope = FrontendProfile.createNewWithScope()) {
       ctx.getQueryCtx().getClient_request().setStmt(stmt);
       StatementBase parsedStmt = Parser.parse(stmt, ctx.getQueryOptions());
+      User user = new User(TSessionStateUtil.getEffectiveUser(ctx.getQueryCtx().session));
       StmtMetadataLoader mdLoader =
-          new StmtMetadataLoader(fe, ctx.getQueryCtx().session.database, null);
+          new StmtMetadataLoader(fe, ctx.getQueryCtx().session.database, null, user);
       StmtTableCache stmtTableCache = mdLoader.loadTables(parsedStmt);
       return ctx.analyzeAndAuthorize(parsedStmt, stmtTableCache, fe.getAuthzChecker());
     }