You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/09/18 05:44:14 UTC

[21/29] kylin git commit: KYLIN-2761, delete table ACL when unloading table.

KYLIN-2761, delete table ACL when unloading table.

minor, fix some typos


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7640c946
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7640c946
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7640c946

Branch: refs/heads/security_update
Commit: 7640c946c7d9fbb5dac49c939b993030e5f90063
Parents: 4237d72
Author: tttMelody <24...@qq.com>
Authored: Sun Sep 17 17:14:58 2017 +0800
Committer: YIMINGXU <10...@qq.com>
Committed: Sun Sep 17 08:29:20 2017 -0500

----------------------------------------------------------------------
 .../apache/kylin/common/KylinConfigBase.java    |  4 +
 .../main/resources/kylin-defaults.properties    |  1 +
 .../org/apache/kylin/metadata/acl/TableACL.java | 15 ++++
 .../kylin/metadata/acl/TableACLManager.java     |  7 ++
 .../apache/kylin/metadata/acl/TableACLTest.java | 18 +++++
 .../query/security/QueryInterceptUtil.java      |  1 -
 .../kylin/query/security/TableLevelACL.java     | 79 --------------------
 .../apache/kylin/query/util/PushDownUtil.java   | 18 +++--
 .../kylin/rest/controller/TableController.java  |  6 ++
 .../kylin/rest/service/TableACLService.java     |  5 ++
 .../apache/kylin/rest/util/ValidateUtil.java    | 12 +--
 .../rest/util/MultiNodeManagerTestBase.java     |  2 +-
 12 files changed, 75 insertions(+), 93 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ab19d2a..15c2843 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1104,6 +1104,10 @@ abstract public class KylinConfigBase implements Serializable {
         return StringUtils.isNotEmpty(getPushDownRunnerClassName());
     }
 
+    public boolean isPushDownUpdateEnabled() {
+        return Boolean.parseBoolean(this.getOptional("kylin.query.pushdown.update-enabled", "false"));
+    }
+
     public String getPushDownRunnerClassName() {
         return getOptional("kylin.query.pushdown.runner-class-name", "");
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index e818dc1..c9ad882 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -247,6 +247,7 @@ kylin.engine.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
 
 #kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl
 
+#kylin.query.pushdown.update-enabled = false
 #kylin.query.pushdown.jdbc.url=jdbc:hive2://sandbox:10000/default
 #kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver
 #kylin.query.pushdown.jdbc.username=hive

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
index 89ef108..0b3ba4d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.acl;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -103,6 +104,20 @@ public class TableACL extends RootPersistentEntity {
         return this;
     }
 
+    public TableACL deleteByTbl(String table) {
+        Iterator<Map.Entry<String, TableBlackList>> it = userTableBlackList.entrySet().iterator();
+        while (it.hasNext()) {
+            Map.Entry<String, TableBlackList> entry = it.next();
+            TableBlackList tableBlackList = entry.getValue();
+            tableBlackList.removeTbl(table);
+            if (tableBlackList.isEmpty()) {
+                it.remove();
+            }
+        }
+
+        return this;
+    }
+
     private void checkUserHasACL(String username) {
         if (userTableBlackList.get(username) == null || userTableBlackList.get(username).isEmpty()) {
             throw new RuntimeException("Operation fail, can not grant user table query permission.User:" + username

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
index 7bea743..4804cef 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
@@ -164,4 +164,11 @@ public class TableACLManager {
         getStore().putResource(path, tableACL, System.currentTimeMillis(), TABLE_ACL_SERIALIZER);
         tableACLMap.put(project, tableACL);
     }
+
+    public void deleteTableACLByTbl(String project, String table) throws IOException {
+        String path = DIR_PREFIX + project;
+        TableACL tableACL = getTableACL(project).deleteByTbl(table);
+        getStore().putResource(path, tableACL, System.currentTimeMillis(), TABLE_ACL_SERIALIZER);
+        tableACLMap.put(project, tableACL);
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java
index 4ad934d..174496e 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java
@@ -44,6 +44,24 @@ public class TableACLTest {
     }
 
     @Test
+    public void testDelTableACLByTable() {
+        //delete specific table's ACL
+        TableACL tableACL = new TableACL();
+        tableACL.add("u1", "t1");
+        tableACL.add("u2", "t1");
+        tableACL.add("u2", "t2");
+        tableACL.add("u2", "t3");
+        tableACL.add("u3", "t3");
+        tableACL.deleteByTbl("t1");
+
+        TableACL expected = new TableACL();
+        expected.add("u2", "t2");
+        expected.add("u2", "t3");
+        expected.add("u3", "t3");
+        Assert.assertEquals(expected, tableACL);
+    }
+
+    @Test
     public void testDeleteToEmpty() {
         TableACL tableACL = new TableACL();
         tableACL.add("u1", "t1");

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
index ecfa8da..b6a97e5 100644
--- a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
@@ -74,7 +74,6 @@ public class QueryInterceptUtil {
                 //computed column
                 if (columnDesc.isComputedColumnn()) {
                     allColWithTblAndSchema.addAll(getCCUsedCols(project, columnDesc));
-                    continue;
                 }
                 //normal column
                 allColWithTblAndSchema.add(tblColRef.getColumWithTableAndSchema());

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/query/src/main/java/org/apache/kylin/query/security/TableLevelACL.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/security/TableLevelACL.java b/query/src/main/java/org/apache/kylin/query/security/TableLevelACL.java
deleted file mode 100644
index c1b67b0..0000000
--- a/query/src/main/java/org/apache/kylin/query/security/TableLevelACL.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.security;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.query.relnode.OLAPContext;
-
-public class TableLevelACL {
-    public static void tableFilter(List<OLAPContext> contexts, List<String> tableBlackList) {
-        Set<String> tableWithSchema = getTableWithSchema(contexts);
-        for (String tbl : tableBlackList) {
-            if (tableWithSchema.contains(tbl.toUpperCase())) {
-//                throw new kylin.AccessDeniedException("table:" + tbl);
-                System.out.println("Access table:" + tbl + " denied");
-            }
-        }
-    }
-
-    public static void columnFilter(List<OLAPContext> contexts, List<String> columnBlackList) {
-        List<String> allColWithTblAndSchema = getAllColWithTblAndSchema(contexts);
-        for (String tbl : columnBlackList) {
-            if (allColWithTblAndSchema.contains(tbl.toUpperCase())) {
-//                throw new kylin.AccessDeniedException("table:" + tbl);
-                System.out.println("Access table:" + tbl + " denied");
-            }
-        }
-    }
-
-    public static List<String> getAllColWithTblAndSchema(List<OLAPContext> contexts) {
-        // all columns with table and DB. Like DB.TABLE.COLUMN
-        List<String> allColWithTblAndSchema = new ArrayList<>();
-        for (OLAPContext context : contexts) {
-            Set<TblColRef> allColumns = context.allColumns;
-            for (TblColRef tblColRef : allColumns) {
-                allColWithTblAndSchema.add(tblColRef.getColumWithTableAndSchema());
-            }
-        }
-        return allColWithTblAndSchema;
-    }
-
-    public static Set<String> getTableWithSchema(List<OLAPContext> contexts) {
-        // all tables with DB, Like DB.TABLE
-        Set<String> tableWithSchema = new HashSet<>();
-        for (OLAPContext context : contexts) {
-            Set<TblColRef> allColumns = context.allColumns;
-            for (TblColRef tblColRef : allColumns) {
-                tableWithSchema.add(tblColRef.getTableWithSchema());
-            }
-        }
-        return tableWithSchema;
-    }
-
-    public static List<String> mockTableBlackList() {
-        List<String> blackList = new ArrayList<>();
-        blackList.add("DEFAULT.KYLIN_SALES");
-        return blackList;
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 50b2136..a9f6c5c 100644
--- a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -122,7 +122,8 @@ public class PushDownUtil {
 
         if (isSelect) {
             runner.executeQuery(sql, returnRows, returnColumnMeta);
-        } else {
+        }
+        if (!isSelect && kylinConfig.isPushDownUpdateEnabled()) {
             runner.executeUpdate(sql);
         }
         return Pair.newPair(returnRows, returnColumnMeta);
@@ -130,12 +131,17 @@ public class PushDownUtil {
 
     private static boolean isExpectedCause(SQLException sqlException) {
         Preconditions.checkArgument(sqlException != null);
-
         Throwable rootCause = ExceptionUtils.getRootCause(sqlException);
-        return rootCause != null && //
-                (rootCause instanceof NoRealizationFoundException //
-                        || rootCause instanceof SqlValidatorException // 
-                        || rootCause instanceof RoutingIndicatorException);
+
+        boolean isPushDownUpdateEnabled = KylinConfig.getInstanceFromEnv().isPushDownUpdateEnabled();
+        if (!isPushDownUpdateEnabled) {
+            return rootCause != null && rootCause instanceof NoRealizationFoundException;
+        } else {
+            return (rootCause != null
+                    && (rootCause instanceof NoRealizationFoundException
+                    || rootCause instanceof SqlValidatorException
+                    || rootCause instanceof RoutingIndicatorException));
+        }
     }
 
     static String schemaCompletion(String inputSql, String schema) throws SqlParseException {

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index d7b4dcb..7a0643e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -31,6 +31,7 @@ import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.exception.NotFoundException;
 import org.apache.kylin.rest.request.CardinalityRequest;
 import org.apache.kylin.rest.request.HiveTableRequest;
+import org.apache.kylin.rest.service.TableACLService;
 import org.apache.kylin.rest.service.TableService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -60,6 +61,10 @@ public class TableController extends BasicController {
     @Qualifier("tableService")
     private TableService tableService;
 
+    @Autowired
+    @Qualifier("TableAclService")
+    private TableACLService tableACLService;
+
     /**
      * Get available table list of the project
      *
@@ -130,6 +135,7 @@ public class TableController extends BasicController {
         Map<String, String[]> result = new HashMap<String, String[]>();
         try {
             for (String tableName : tables.split(",")) {
+                tableACLService.deleteFromTableBlackListByTbl(project, tableName);
                 if (tableService.unloadHiveTable(tableName, project)) {
                     unLoadSuccess.add(tableName);
                 } else {

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
index 702d0c3..b12ecab 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
@@ -78,4 +78,9 @@ public class TableACLService extends BasicService {
         aclEvaluate.checkProjectAdminPermission(project);
         getTableACLManager().deleteTableACL(project, username);
     }
+
+    public void deleteFromTableBlackListByTbl(String project, String table) throws IOException {
+        aclEvaluate.checkProjectAdminPermission(project);
+        getTableACLManager().deleteTableACLByTbl(project, table);
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
index fc106ce..abb326f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
@@ -19,11 +19,11 @@
 package org.apache.kylin.rest.util;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.persistence.AclEntity;
@@ -73,7 +73,7 @@ public class ValidateUtil {
 
     public void validateTable(String project, String table) throws IOException {
         List<TableDesc> tableDescs = tableService.getTableDescByProject(project, false);
-        List<String> tables = new ArrayList<>();
+        Set<String> tables = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
         for (TableDesc tableDesc : tableDescs) {
             tables.add(tableDesc.getDatabase() + "." + tableDesc.getName());
         }
@@ -85,7 +85,7 @@ public class ValidateUtil {
 
     public void validateColumn(String project, String table, Collection<String> columns) throws IOException {
         Preconditions.checkState(columns != null && columns.size() > 0);
-        List<String> cols = getAllColumns(project, table);
+        Set<String> cols = getAllColumns(project, table);
         for (String c : columns) {
             if (!cols.contains(c)) {
                 throw new RuntimeException("Operation failed, column:" + c + " not exists");
@@ -93,13 +93,13 @@ public class ValidateUtil {
         }
     }
 
-    private List<String> getAllColumns(String project, String table) throws IOException {
+    private Set<String> getAllColumns(String project, String table) throws IOException {
         List<TableDesc> tableDescByProject = tableService.getTableDescByProject(project, true);
-        List<String> cols = new ArrayList<>();
+        Set<String> cols = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
 
         for (TableDesc tableDesc : tableDescByProject) {
             String tbl = tableDesc.getDatabase() + "." + tableDesc.getName();
-            if (tbl.equals(table)) {
+            if (tbl.equalsIgnoreCase(table)) {
                 for (ColumnDesc column : tableDesc.getColumns()) {
                     cols.add(column.getName());
                 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java b/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java
index 20baeca..9610f2f 100644
--- a/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java
@@ -45,7 +45,7 @@ public class MultiNodeManagerTestBase extends LocalFileMetadataTestCase {
     @Before
     public void setup() throws Exception {
         staticCreateTestMetadata();
-
+        System.clearProperty("kylin.server.cluster-servers");
         int port = CheckUtil.randomAvailablePort(40000, 50000);
         logger.info("Chosen port for CacheServiceTest is " + port);
         configA = KylinConfig.getInstanceFromEnv();