You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ng...@apache.org on 2022/08/15 22:13:05 UTC
[hive] branch master updated: HIVE-26246: Filter out results 'show connectors' on HMS client-side (#3304)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5a50af46b87 HIVE-26246: Filter out results 'show connectors' on HMS client-side (#3304)
5a50af46b87 is described below
commit 5a50af46b877e510d67d74b164393956aeac871d
Author: Butao Zhang <97...@users.noreply.github.com>
AuthorDate: Tue Aug 16 06:12:54 2022 +0800
HIVE-26246: Filter out results 'show connectors' on HMS client-side (#3304)
---
.../plugin/AuthorizationMetaStoreFilterHook.java | 10 ++++---
.../plugin/HivePrivilegeObjectUtils.java | 14 ++++++++++
.../plugin/metastore/HiveMetaStoreAuthorizer.java | 5 ++++
.../metastore/DefaultMetaStoreFilterHookImpl.java | 5 ++++
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 3 ++-
.../hadoop/hive/metastore/MetaStoreFilterHook.java | 7 +++++
.../hadoop/hive/metastore/utils/FilterUtils.java | 2 +-
.../hadoop/hive/metastore/TestFilterHooks.java | 31 ++++++++++++++++++++++
8 files changed, 72 insertions(+), 5 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
index 3bacfb458b1..e40297620c2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/AuthorizationMetaStoreFilterHook.java
@@ -49,7 +49,7 @@ public class AuthorizationMetaStoreFilterHook extends DefaultMetaStoreFilterHook
public List<String> filterTableNames(String catName, String dbName, List<String> tableList)
throws MetaException {
List<HivePrivilegeObject> listObjs = getHivePrivObjects(dbName, tableList);
- return getTableNames(getFilteredObjects(listObjs));
+ return getFilteredObjectNames(getFilteredObjects(listObjs));
}
@Override
public List<Table> filterTables(List<Table> tableList) throws MetaException {
@@ -98,7 +98,7 @@ public class AuthorizationMetaStoreFilterHook extends DefaultMetaStoreFilterHook
return tnames;
}
- private List<String> getTableNames(List<HivePrivilegeObject> filteredObjects) {
+ private List<String> getFilteredObjectNames(List<HivePrivilegeObject> filteredObjects) {
List<String> tnames = new ArrayList<String>();
for(HivePrivilegeObject obj : filteredObjects) {
tnames.add(obj.getObjectName());
@@ -153,6 +153,10 @@ public class AuthorizationMetaStoreFilterHook extends DefaultMetaStoreFilterHook
.filter(e -> filteredTableNames.contains(e.getTableName())).collect(Collectors.toList());
}
-
+ @Override
+ public List<String> filterDataConnectors(List<String> dcList) throws MetaException {
+ List<HivePrivilegeObject> listObjs = HivePrivilegeObjectUtils.getHivePrivDcObjects(dcList);
+ return getFilteredObjectNames(getFilteredObjects(listObjs));
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java
index 74a53f24eba..4ab7870e7b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HivePrivilegeObjectUtils.java
@@ -48,4 +48,18 @@ public class HivePrivilegeObjectUtils {
}
+ /**
+ * Convert list of dcnames into list of HivePrivilegeObject
+ * @param dcList
+ * @return
+ */
+ public static List<HivePrivilegeObject> getHivePrivDcObjects(List<String> dcList) {
+ List<HivePrivilegeObject> objs = new ArrayList<HivePrivilegeObject>();
+ for (String dcname : dcList) {
+ objs.add(new HivePrivilegeObject(HivePrivilegeObjectType.DATACONNECTOR, null, dcname));
+ }
+ return objs;
+
+ }
+
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java
index 969254ab1ae..c276e6cccb8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/metastore/HiveMetaStoreAuthorizer.java
@@ -233,6 +233,11 @@ public class HiveMetaStoreAuthorizer extends MetaStorePreEventListener implement
return list;
}
+ @Override
+ public List<String> filterDataConnectors(List<String> dcList) throws MetaException {
+ return dcList;
+ }
+
private List<String> filterDatabaseObjects(HiveMetaStoreAuthzInfo hiveMetaStoreAuthzInfo) throws MetaException {
List<String> ret = null;
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
index 36a82c5149a..3af78502040 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/DefaultMetaStoreFilterHookImpl.java
@@ -90,4 +90,9 @@ public class DefaultMetaStoreFilterHookImpl implements MetaStoreFilterHook {
return partitionNames;
}
+ @Override
+ public List<String> filterDataConnectors(List<String> dcList) throws MetaException {
+ return dcList;
+ }
+
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index ac6979c20ff..93229b95007 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -1342,7 +1342,8 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
*/
@Override
public List<String> getAllDataConnectorNames() throws MetaException, TException {
- return client.get_dataconnectors();
+ List<String> connectorNames = client.get_dataconnectors();
+ return FilterUtils.filterDataConnectorsIfEnabled(isClientFilterEnabled, filterHook, connectorNames);
}
/**
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
index 2f48aed37f1..e2da15e54f2 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
@@ -143,5 +143,12 @@ public interface MetaStoreFilterHook {
*/
List<String> filterPartitionNames(String catName, String dbName, String tblName,
List<String> partitionNames) throws MetaException;
+
+ /**
+ * Filter given list of data connectors
+ * @param dcList
+ * @return List of filtered Dc names
+ */
+ List<String> filterDataConnectors(List<String> dcList) throws MetaException;
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java
index c125f6993e4..d1ebe0aeaaa 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FilterUtils.java
@@ -110,7 +110,7 @@ public class FilterUtils {
List<String> connectorNames) throws MetaException {
if (isFilterEnabled) {
- return filterHook.filterDatabases(connectorNames); // TODO add a new ATZ call
+ return filterHook.filterDataConnectors(connectorNames);
}
return connectorNames;
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
index 6e37c3b13a7..3f91c92e1cb 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.List;
import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.DataConnector;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
@@ -141,6 +142,14 @@ public class TestFilterHooks {
}
return partitionNames;
}
+
+ @Override
+ public List<String> filterDataConnectors(List<String> dcList) throws MetaException {
+ if (blockResults) {
+ return new ArrayList<>();
+ }
+ return dcList;
+ }
}
protected static HiveMetaStoreClient client;
@@ -153,6 +162,12 @@ public class TestFilterHooks {
private static String DBNAME2 = "testdb2";
private static final String TAB1 = "tab1";
private static final String TAB2 = "tab2";
+ private static String DCNAME1 = "test_connector1";
+ private static String DCNAME2 = "test_connector2";
+ private static String mysql_type = "mysql";
+ private static String mysql_url = "jdbc:mysql://localhost:3306/hive";
+ private static String postgres_type = "postgres";
+ private static String postgres_url = "jdbc:postgresql://localhost:5432";
protected HiveMetaStoreClient createClient(Configuration metaStoreConf) throws Exception {
@@ -210,6 +225,8 @@ public class TestFilterHooks {
client.dropDatabase(DBNAME1, true, true, true);
client.dropDatabase(DBNAME2, true, true, true);
+ client.dropDataConnector(DCNAME1, true, true);
+ client.dropDataConnector(DCNAME2, true, true);
Database db1 = new DatabaseBuilder()
.setName(DBNAME1)
.setCatalogName(Warehouse.DEFAULT_CATALOG_NAME)
@@ -238,6 +255,10 @@ public class TestFilterHooks {
.inTable(tab2)
.addValue("value2")
.addToTable(client, conf);
+ DataConnector dc1 = new DataConnector(DCNAME1, mysql_type, mysql_url);
+ DataConnector dc2 = new DataConnector(DCNAME2, postgres_type, postgres_url);
+ client.createDataConnector(dc1);
+ client.createDataConnector(dc2);
TestTxnDbUtil.cleanDb(conf);
TestTxnDbUtil.prepDb(conf);
@@ -271,6 +292,8 @@ public class TestFilterHooks {
assertEquals(1, client.getPartitionsByNames(DBNAME1, TAB2, Lists.newArrayList("name=value1")).size());
assertEquals(2, client.showCompactions().getCompacts().size());
+
+ assertEquals(2, client.getAllDataConnectorNames().size());
}
/**
@@ -318,6 +341,8 @@ public class TestFilterHooks {
assertEquals(1, client.getPartitionsByNames(DBNAME1, TAB2, Lists.newArrayList("name=value1")).size());
assertEquals(2, client.showCompactions().getCompacts().size());
+
+ assertEquals(2, client.getAllDataConnectorNames().size());
}
/**
@@ -336,6 +361,7 @@ public class TestFilterHooks {
testFilterForTables(false);
testFilterForPartition(false);
testFilterForCompaction();
+ testFilterForDataConnector();
}
protected void testFilterForDb(boolean filterAtServer) throws Exception {
@@ -404,4 +430,9 @@ public class TestFilterHooks {
protected void testFilterForCompaction() throws Exception {
assertEquals(0, client.showCompactions().getCompacts().size());
}
+
+ protected void testFilterForDataConnector() throws Exception {
+ assertNotNull(client.getDataConnector(DCNAME1));
+ assertEquals(0, client.getAllDataConnectorNames().size());
+ }
}