You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ma...@apache.org on 2019/11/27 10:54:17 UTC
[hive] branch master updated: HIVE-22512 : Use direct SQL to fetch
column privileges in refreshPrivileges. (Ashutosh Bapat reviewed by Mahesh
Kumar Behera)
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2b34ad7 HIVE-22512 : Use direct SQL to fetch column privileges in refreshPrivileges. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
2b34ad7 is described below
commit 2b34ad7facc1aed74307f5c7991019116b043346
Author: Ashutosh Bapat <ab...@cloudera.com>
AuthorDate: Wed Nov 27 16:22:29 2019 +0530
HIVE-22512 : Use direct SQL to fetch column privileges in refreshPrivileges. (Ashutosh Bapat reviewed by Mahesh Kumar Behera)
Signed-off-by: Mahesh Kumar Behera <ma...@apache.org>
---
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 94 +++++++++++++++++++++-
.../apache/hadoop/hive/metastore/ObjectStore.java | 40 ++++++++-
2 files changed, 129 insertions(+), 5 deletions(-)
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 54fe1eb..9f2b436 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -55,10 +55,14 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.GetPartitionsFilterSpec;
+import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
+import org.apache.hadoop.hive.metastore.api.HiveObjectRef;
+import org.apache.hadoop.hive.metastore.api.HiveObjectType;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
@@ -159,8 +163,8 @@ class MetaStoreDirectSql {
private String DBS, TBLS, PARTITIONS, DATABASE_PARAMS, PARTITION_PARAMS, SORT_COLS, SD_PARAMS,
SDS, SERDES, SKEWED_STRING_LIST_VALUES, SKEWED_VALUES, BUCKETING_COLS, SKEWED_COL_NAMES,
SKEWED_COL_VALUE_LOC_MAP, COLUMNS_V2, PARTITION_KEYS, SERDE_PARAMS, PART_COL_STATS, KEY_CONSTRAINTS,
- TAB_COL_STATS, PARTITION_KEY_VALS, PART_PRIVS, PART_COL_PRIVS, SKEWED_STRING_LIST, CDS;
-
+ TAB_COL_STATS, PARTITION_KEY_VALS, PART_PRIVS, PART_COL_PRIVS, SKEWED_STRING_LIST, CDS,
+ TBL_COL_PRIVS;
public MetaStoreDirectSql(PersistenceManager pm, Configuration conf, String schema) {
this.pm = pm;
@@ -1280,6 +1284,92 @@ class MetaStoreDirectSql {
return result;
}
+ public List<HiveObjectPrivilege> getTableAllColumnGrants(String catName, String dbName,
+ String tableName, String authorizer) throws MetaException {
+ Query query = null;
+
+ // These constants should match the SELECT clause of the query.
+ final int authorizerIndex = 0;
+ final int columnNameIndex = 1;
+ final int createTimeIndex = 2;
+ final int grantOptionIndex = 3;
+ final int grantorIndex = 4;
+ final int grantorTypeIndex = 5;
+ final int principalNameIndex = 6;
+ final int principalTypeIndex = 7;
+ final int privilegeIndex = 8;
+
+ // Retrieve the privileges from the object store. Just grab only the required fields.
+ String queryText = "select " +
+ TBL_COL_PRIVS + ".\"AUTHORIZER\", " +
+ TBL_COL_PRIVS + ".\"COLUMN_NAME\", " +
+ TBL_COL_PRIVS + ".\"CREATE_TIME\", " +
+ TBL_COL_PRIVS + ".\"GRANT_OPTION\", " +
+ TBL_COL_PRIVS + ".\"GRANTOR\", " +
+ TBL_COL_PRIVS + ".\"GRANTOR_TYPE\", " +
+ TBL_COL_PRIVS + ".\"PRINCIPAL_NAME\", " +
+ TBL_COL_PRIVS + ".\"PRINCIPAL_TYPE\", " +
+ TBL_COL_PRIVS + ".\"TBL_COL_PRIV\" " +
+ "FROM " + TBL_COL_PRIVS + " JOIN " + TBLS +
+ " ON " + TBL_COL_PRIVS + ".\"TBL_ID\" = " + TBLS + ".\"TBL_ID\"" +
+ " JOIN " + DBS + " ON " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" " +
+ " WHERE " + TBLS + ".\"TBL_NAME\" = ?" +
+ " AND " + DBS + ".\"NAME\" = ?" +
+ " AND " + DBS + ".\"CTLG_NAME\" = ?";
+
+ // Build the parameters, they should match the WHERE clause of the query.
+ int numParams = authorizer != null ? 4 : 3;
+ Object[] params = new Object[numParams];
+ params[0] = tableName;
+ params[1] = dbName;
+ params[2] = catName;
+ if (authorizer != null) {
+ queryText = queryText + " AND " + TBL_COL_PRIVS + ".\"AUTHORIZER\" = ?";
+ params[3] = authorizer;
+ }
+
+ // Collect the results into a list that the caller can consume.
+ List<HiveObjectPrivilege> result = new ArrayList<>();
+ final boolean doTrace = LOG.isDebugEnabled();
+ long start = doTrace ? System.nanoTime() : 0;
+ query = pm.newQuery("javax.jdo.query.SQL", queryText);
+ try {
+ List<Object[]> queryResult = MetastoreDirectSqlUtils.ensureList(
+ executeWithArray(query, params, queryText));
+ long end = doTrace ? System.nanoTime() : 0;
+ MetastoreDirectSqlUtils.timingTrace(doTrace, queryText, start, end);
+
+ // If there is some result convert it into HivePrivilege bag and return.
+ for (Object[] privLine : queryResult) {
+ String privAuthorizer = MetastoreDirectSqlUtils.extractSqlString(privLine[authorizerIndex]);
+ String principalName = MetastoreDirectSqlUtils.extractSqlString(privLine[principalNameIndex]);
+ PrincipalType ptype = PrincipalType.valueOf(
+ MetastoreDirectSqlUtils.extractSqlString(privLine[principalTypeIndex]));
+ String columnName = MetastoreDirectSqlUtils.extractSqlString(privLine[columnNameIndex]);
+ String privilege = MetastoreDirectSqlUtils.extractSqlString(privLine[privilegeIndex]);
+ int createTime = MetastoreDirectSqlUtils.extractSqlInt(privLine[createTimeIndex]);
+ String grantor = MetastoreDirectSqlUtils.extractSqlString(privLine[grantorIndex]);
+ PrincipalType grantorType =
+ PrincipalType.valueOf(
+ MetastoreDirectSqlUtils.extractSqlString(privLine[grantorTypeIndex]));
+ boolean grantOption = MetastoreDirectSqlUtils.extractSqlBoolean(privLine[grantOptionIndex]);
+
+ HiveObjectRef objectRef = new HiveObjectRef(HiveObjectType.COLUMN, dbName, tableName, null,
+ columnName);
+ objectRef.setCatName(catName);
+ PrivilegeGrantInfo grantInfo = new PrivilegeGrantInfo(privilege, createTime, grantor,
+ grantorType, grantOption);
+
+ result.add(new HiveObjectPrivilege(objectRef, principalName, ptype, grantInfo,
+ privAuthorizer));
+ }
+ } finally {
+ query.closeAll();
+ }
+
+ return result;
+ }
+
public AggrStats aggrColStatsForPartitions(String catName, String dbName, String tableName,
List<String> partNames, List<String> colNames, String engine,
boolean useDensityFunctionForNDVEstimation, double ndvTuner, boolean enableBitVector)
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index fb9d870..5525109 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -6452,8 +6452,8 @@ public class ObjectStore implements RawStore, Configurable {
break;
case COLUMN:
Preconditions.checkArgument(objToRefresh.getColumnName()==null, "columnName must be null");
- grants = convertTableCols(listTableAllColumnGrants(catName,
- objToRefresh.getDbName(), objToRefresh.getObjectName(), authorizer));
+ grants = getTableAllColumnGrants(catName, objToRefresh.getDbName(),
+ objToRefresh.getObjectName(), authorizer);
break;
default:
throw new MetaException("Unexpected object type " + objToRefresh.getObjectType());
@@ -6471,18 +6471,24 @@ public class ObjectStore implements RawStore, Configurable {
}
}
if (!revokePrivilegeSet.isEmpty()) {
+ LOG.debug("Found " + revokePrivilegeSet.size() + " new revoke privileges to be synced.");
PrivilegeBag remainingRevokePrivileges = new PrivilegeBag();
for (HiveObjectPrivilege revokePrivilege : revokePrivilegeSet) {
remainingRevokePrivileges.addToPrivileges(revokePrivilege);
}
revokePrivileges(remainingRevokePrivileges, false);
+ } else {
+ LOG.debug("No new revoke privileges are required to be synced.");
}
if (!grantPrivilegeSet.isEmpty()) {
+ LOG.debug("Found " + grantPrivilegeSet.size() + " new grant privileges to be synced.");
PrivilegeBag remainingGrantPrivileges = new PrivilegeBag();
for (HiveObjectPrivilege grantPrivilege : grantPrivilegeSet) {
remainingGrantPrivileges.addToPrivileges(grantPrivilege);
}
grantPrivileges(remainingGrantPrivileges);
+ } else {
+ LOG.debug("No new grant privileges are required to be synced.");
}
committed = commitTransaction();
} finally {
@@ -6493,6 +6499,30 @@ public class ObjectStore implements RawStore, Configurable {
return committed;
}
+ private List<HiveObjectPrivilege> getTableAllColumnGrants(String catName, String dbName,
+ String tableName, String authorizer)
+ throws MetaException, NoSuchObjectException {
+ return new GetListHelper<HiveObjectPrivilege>(normalizeIdentifier(catName),
+ normalizeIdentifier(dbName), normalizeIdentifier(tableName), true, true) {
+
+ @Override
+ protected String describeResult() {
+ return "Table column privileges.";
+ }
+
+ @Override
+ protected List<HiveObjectPrivilege> getSqlResult(GetHelper<List<HiveObjectPrivilege>> ctx)
+ throws MetaException {
+ return directSql.getTableAllColumnGrants(catName, dbName, tblName, authorizer);
+ }
+
+ @Override
+ protected List<HiveObjectPrivilege> getJdoResult(GetHelper<List<HiveObjectPrivilege>> ctx) {
+ return convertTableCols(listTableAllColumnGrants(catName, dbName, tblName, authorizer));
+ }
+ }.run(false);
+ }
+
public List<MRoleMap> listMRoleMembers(String roleName) {
boolean success = false;
Query query = null;
@@ -6864,12 +6894,16 @@ public class ObjectStore implements RawStore, Configurable {
query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
mPrivs = (List<MTableColumnPrivilege>) query.executeWithArray(tableName, dbName, catName);
}
+ LOG.debug("Query to obtain objects for listTableAllColumnGrants finished");
pm.retrieveAll(mPrivs);
+ LOG.debug("RetrieveAll on all the objects for listTableAllColumnGrants finished");
success = commitTransaction();
+ LOG.debug("Transaction running query to obtain objects for listTableAllColumnGrants " +
+ "committed");
mTblColPrivilegeList.addAll(mPrivs);
- LOG.debug("Done retrieving all objects for listTableAllColumnGrants");
+ LOG.debug("Done retrieving " + mPrivs.size() + " objects for listTableAllColumnGrants");
} finally {
rollbackAndCleanup(success, query);
}