You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/09/18 05:44:14 UTC
[21/29] kylin git commit: KYLIN-2761,
delete table ACL when unloading table.
KYLIN-2761, delete table ACL when unloading table.
minor, fix some typos
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7640c946
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7640c946
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7640c946
Branch: refs/heads/security_update
Commit: 7640c946c7d9fbb5dac49c939b993030e5f90063
Parents: 4237d72
Author: tttMelody <24...@qq.com>
Authored: Sun Sep 17 17:14:58 2017 +0800
Committer: YIMINGXU <10...@qq.com>
Committed: Sun Sep 17 08:29:20 2017 -0500
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../main/resources/kylin-defaults.properties | 1 +
.../org/apache/kylin/metadata/acl/TableACL.java | 15 ++++
.../kylin/metadata/acl/TableACLManager.java | 7 ++
.../apache/kylin/metadata/acl/TableACLTest.java | 18 +++++
.../query/security/QueryInterceptUtil.java | 1 -
.../kylin/query/security/TableLevelACL.java | 79 --------------------
.../apache/kylin/query/util/PushDownUtil.java | 18 +++--
.../kylin/rest/controller/TableController.java | 6 ++
.../kylin/rest/service/TableACLService.java | 5 ++
.../apache/kylin/rest/util/ValidateUtil.java | 12 +--
.../rest/util/MultiNodeManagerTestBase.java | 2 +-
12 files changed, 75 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ab19d2a..15c2843 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1104,6 +1104,10 @@ abstract public class KylinConfigBase implements Serializable {
return StringUtils.isNotEmpty(getPushDownRunnerClassName());
}
+ public boolean isPushDownUpdateEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.query.pushdown.update-enabled", "false"));
+ }
+
public String getPushDownRunnerClassName() {
return getOptional("kylin.query.pushdown.runner-class-name", "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index e818dc1..c9ad882 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -247,6 +247,7 @@ kylin.engine.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
#kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl
+#kylin.query.pushdown.update-enabled = false
#kylin.query.pushdown.jdbc.url=jdbc:hive2://sandbox:10000/default
#kylin.query.pushdown.jdbc.driver=org.apache.hive.jdbc.HiveDriver
#kylin.query.pushdown.jdbc.username=hive
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
index 89ef108..0b3ba4d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.acl;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -103,6 +104,20 @@ public class TableACL extends RootPersistentEntity {
return this;
}
+ public TableACL deleteByTbl(String table) {
+ Iterator<Map.Entry<String, TableBlackList>> it = userTableBlackList.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry<String, TableBlackList> entry = it.next();
+ TableBlackList tableBlackList = entry.getValue();
+ tableBlackList.removeTbl(table);
+ if (tableBlackList.isEmpty()) {
+ it.remove();
+ }
+ }
+
+ return this;
+ }
+
private void checkUserHasACL(String username) {
if (userTableBlackList.get(username) == null || userTableBlackList.get(username).isEmpty()) {
throw new RuntimeException("Operation fail, can not grant user table query permission.User:" + username
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
index 7bea743..4804cef 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACLManager.java
@@ -164,4 +164,11 @@ public class TableACLManager {
getStore().putResource(path, tableACL, System.currentTimeMillis(), TABLE_ACL_SERIALIZER);
tableACLMap.put(project, tableACL);
}
+
+ public void deleteTableACLByTbl(String project, String table) throws IOException {
+ String path = DIR_PREFIX + project;
+ TableACL tableACL = getTableACL(project).deleteByTbl(table);
+ getStore().putResource(path, tableACL, System.currentTimeMillis(), TABLE_ACL_SERIALIZER);
+ tableACLMap.put(project, tableACL);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java
index 4ad934d..174496e 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/acl/TableACLTest.java
@@ -44,6 +44,24 @@ public class TableACLTest {
}
@Test
+ public void testDelTableACLByTable() {
+ //delete specific table's ACL
+ TableACL tableACL = new TableACL();
+ tableACL.add("u1", "t1");
+ tableACL.add("u2", "t1");
+ tableACL.add("u2", "t2");
+ tableACL.add("u2", "t3");
+ tableACL.add("u3", "t3");
+ tableACL.deleteByTbl("t1");
+
+ TableACL expected = new TableACL();
+ expected.add("u2", "t2");
+ expected.add("u2", "t3");
+ expected.add("u3", "t3");
+ Assert.assertEquals(expected, tableACL);
+ }
+
+ @Test
public void testDeleteToEmpty() {
TableACL tableACL = new TableACL();
tableACL.add("u1", "t1");
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
index ecfa8da..b6a97e5 100644
--- a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptUtil.java
@@ -74,7 +74,6 @@ public class QueryInterceptUtil {
//computed column
if (columnDesc.isComputedColumnn()) {
allColWithTblAndSchema.addAll(getCCUsedCols(project, columnDesc));
- continue;
}
//normal column
allColWithTblAndSchema.add(tblColRef.getColumWithTableAndSchema());
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/query/src/main/java/org/apache/kylin/query/security/TableLevelACL.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/security/TableLevelACL.java b/query/src/main/java/org/apache/kylin/query/security/TableLevelACL.java
deleted file mode 100644
index c1b67b0..0000000
--- a/query/src/main/java/org/apache/kylin/query/security/TableLevelACL.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.query.security;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.query.relnode.OLAPContext;
-
-public class TableLevelACL {
- public static void tableFilter(List<OLAPContext> contexts, List<String> tableBlackList) {
- Set<String> tableWithSchema = getTableWithSchema(contexts);
- for (String tbl : tableBlackList) {
- if (tableWithSchema.contains(tbl.toUpperCase())) {
-// throw new kylin.AccessDeniedException("table:" + tbl);
- System.out.println("Access table:" + tbl + " denied");
- }
- }
- }
-
- public static void columnFilter(List<OLAPContext> contexts, List<String> columnBlackList) {
- List<String> allColWithTblAndSchema = getAllColWithTblAndSchema(contexts);
- for (String tbl : columnBlackList) {
- if (allColWithTblAndSchema.contains(tbl.toUpperCase())) {
-// throw new kylin.AccessDeniedException("table:" + tbl);
- System.out.println("Access table:" + tbl + " denied");
- }
- }
- }
-
- public static List<String> getAllColWithTblAndSchema(List<OLAPContext> contexts) {
- // all columns with table and DB. Like DB.TABLE.COLUMN
- List<String> allColWithTblAndSchema = new ArrayList<>();
- for (OLAPContext context : contexts) {
- Set<TblColRef> allColumns = context.allColumns;
- for (TblColRef tblColRef : allColumns) {
- allColWithTblAndSchema.add(tblColRef.getColumWithTableAndSchema());
- }
- }
- return allColWithTblAndSchema;
- }
-
- public static Set<String> getTableWithSchema(List<OLAPContext> contexts) {
- // all tables with DB, Like DB.TABLE
- Set<String> tableWithSchema = new HashSet<>();
- for (OLAPContext context : contexts) {
- Set<TblColRef> allColumns = context.allColumns;
- for (TblColRef tblColRef : allColumns) {
- tableWithSchema.add(tblColRef.getTableWithSchema());
- }
- }
- return tableWithSchema;
- }
-
- public static List<String> mockTableBlackList() {
- List<String> blackList = new ArrayList<>();
- blackList.add("DEFAULT.KYLIN_SALES");
- return blackList;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 50b2136..a9f6c5c 100644
--- a/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -122,7 +122,8 @@ public class PushDownUtil {
if (isSelect) {
runner.executeQuery(sql, returnRows, returnColumnMeta);
- } else {
+ }
+ if (!isSelect && kylinConfig.isPushDownUpdateEnabled()) {
runner.executeUpdate(sql);
}
return Pair.newPair(returnRows, returnColumnMeta);
@@ -130,12 +131,17 @@ public class PushDownUtil {
private static boolean isExpectedCause(SQLException sqlException) {
Preconditions.checkArgument(sqlException != null);
-
Throwable rootCause = ExceptionUtils.getRootCause(sqlException);
- return rootCause != null && //
- (rootCause instanceof NoRealizationFoundException //
- || rootCause instanceof SqlValidatorException //
- || rootCause instanceof RoutingIndicatorException);
+
+ boolean isPushDownUpdateEnabled = KylinConfig.getInstanceFromEnv().isPushDownUpdateEnabled();
+ if (!isPushDownUpdateEnabled) {
+ return rootCause != null && rootCause instanceof NoRealizationFoundException;
+ } else {
+ return (rootCause != null
+ && (rootCause instanceof NoRealizationFoundException
+ || rootCause instanceof SqlValidatorException
+ || rootCause instanceof RoutingIndicatorException));
+ }
}
static String schemaCompletion(String inputSql, String schema) throws SqlParseException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index d7b4dcb..7a0643e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -31,6 +31,7 @@ import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.request.CardinalityRequest;
import org.apache.kylin.rest.request.HiveTableRequest;
+import org.apache.kylin.rest.service.TableACLService;
import org.apache.kylin.rest.service.TableService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,6 +61,10 @@ public class TableController extends BasicController {
@Qualifier("tableService")
private TableService tableService;
+ @Autowired
+ @Qualifier("TableAclService")
+ private TableACLService tableACLService;
+
/**
* Get available table list of the project
*
@@ -130,6 +135,7 @@ public class TableController extends BasicController {
Map<String, String[]> result = new HashMap<String, String[]>();
try {
for (String tableName : tables.split(",")) {
+ tableACLService.deleteFromTableBlackListByTbl(project, tableName);
if (tableService.unloadHiveTable(tableName, project)) {
unLoadSuccess.add(tableName);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
index 702d0c3..b12ecab 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
@@ -78,4 +78,9 @@ public class TableACLService extends BasicService {
aclEvaluate.checkProjectAdminPermission(project);
getTableACLManager().deleteTableACL(project, username);
}
+
+ public void deleteFromTableBlackListByTbl(String project, String table) throws IOException {
+ aclEvaluate.checkProjectAdminPermission(project);
+ getTableACLManager().deleteTableACLByTbl(project, table);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
index fc106ce..abb326f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
@@ -19,11 +19,11 @@
package org.apache.kylin.rest.util;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.TreeSet;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.persistence.AclEntity;
@@ -73,7 +73,7 @@ public class ValidateUtil {
public void validateTable(String project, String table) throws IOException {
List<TableDesc> tableDescs = tableService.getTableDescByProject(project, false);
- List<String> tables = new ArrayList<>();
+ Set<String> tables = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
for (TableDesc tableDesc : tableDescs) {
tables.add(tableDesc.getDatabase() + "." + tableDesc.getName());
}
@@ -85,7 +85,7 @@ public class ValidateUtil {
public void validateColumn(String project, String table, Collection<String> columns) throws IOException {
Preconditions.checkState(columns != null && columns.size() > 0);
- List<String> cols = getAllColumns(project, table);
+ Set<String> cols = getAllColumns(project, table);
for (String c : columns) {
if (!cols.contains(c)) {
throw new RuntimeException("Operation failed, column:" + c + " not exists");
@@ -93,13 +93,13 @@ public class ValidateUtil {
}
}
- private List<String> getAllColumns(String project, String table) throws IOException {
+ private Set<String> getAllColumns(String project, String table) throws IOException {
List<TableDesc> tableDescByProject = tableService.getTableDescByProject(project, true);
- List<String> cols = new ArrayList<>();
+ Set<String> cols = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
for (TableDesc tableDesc : tableDescByProject) {
String tbl = tableDesc.getDatabase() + "." + tableDesc.getName();
- if (tbl.equals(table)) {
+ if (tbl.equalsIgnoreCase(table)) {
for (ColumnDesc column : tableDesc.getColumns()) {
cols.add(column.getName());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7640c946/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java b/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java
index 20baeca..9610f2f 100644
--- a/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/util/MultiNodeManagerTestBase.java
@@ -45,7 +45,7 @@ public class MultiNodeManagerTestBase extends LocalFileMetadataTestCase {
@Before
public void setup() throws Exception {
staticCreateTestMetadata();
-
+ System.clearProperty("kylin.server.cluster-servers");
int port = CheckUtil.randomAvailablePort(40000, 50000);
logger.info("Chosen port for CacheServiceTest is " + port);
configA = KylinConfig.getInstanceFromEnv();