You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2018/03/31 01:25:35 UTC
[16/44] hive git commit: HIVE-18755 Modifications to the metastore
for catalogs (Alan Gates, reviewed by Thejas Nair)
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java
index 1636d48..80cb1de 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java
@@ -130,10 +130,13 @@ public final class MaterializationsInvalidationCache {
public void run() {
try {
RawStore store = handler.getMS();
- for (String dbName : store.getAllDatabases()) {
- for (Table mv : store.getTableObjectsByName(dbName, store.getTables(dbName, null, TableType.MATERIALIZED_VIEW))) {
- addMaterializedView(mv.getDbName(), mv.getTableName(), ImmutableSet.copyOf(mv.getCreationMetadata().getTablesUsed()),
- mv.getCreationMetadata().getValidTxnList(), OpType.LOAD);
+ for (String catName : store.getCatalogs()) {
+ for (String dbName : store.getAllDatabases(catName)) {
+ for (Table mv : store.getTableObjectsByName(catName, dbName,
+ store.getTables(catName, dbName, null, TableType.MATERIALIZED_VIEW))) {
+ addMaterializedView(mv.getDbName(), mv.getTableName(), ImmutableSet.copyOf(mv.getCreationMetadata().getTablesUsed()),
+ mv.getCreationMetadata().getValidTxnList(), OpType.LOAD);
+ }
}
}
LOG.info("Initialized materializations invalidation cache");
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 0dcf117..997f5fd 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -19,7 +19,9 @@
package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang.StringUtils.join;
+import static org.apache.commons.lang.StringUtils.normalizeSpace;
import static org.apache.commons.lang.StringUtils.repeat;
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
import java.sql.Blob;
import java.sql.Clob;
@@ -311,17 +313,19 @@ class MetaStoreDirectSql {
}
}
- public Database getDatabase(String dbName) throws MetaException{
+ public Database getDatabase(String catName, String dbName) throws MetaException{
Query queryDbSelector = null;
Query queryDbParams = null;
try {
dbName = dbName.toLowerCase();
+ catName = catName.toLowerCase();
String queryTextDbSelector= "select "
+ "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", "
- + "\"OWNER_NAME\", \"OWNER_TYPE\" "
- + "FROM "+ DBS +" where \"NAME\" = ? ";
- Object[] params = new Object[] { dbName };
+ + "\"OWNER_NAME\", \"OWNER_TYPE\", \"CTLG_NAME\" "
+ + "FROM "+ DBS
+ + " where \"NAME\" = ? and \"CTLG_NAME\" = ? ";
+ Object[] params = new Object[] { dbName, catName };
queryDbSelector = pm.newQuery("javax.jdo.query.SQL", queryTextDbSelector);
if (LOG.isTraceEnabled()) {
@@ -370,6 +374,7 @@ class MetaStoreDirectSql {
String type = extractSqlString(dbline[5]);
db.setOwnerType(
(null == type || type.trim().isEmpty()) ? null : PrincipalType.valueOf(type));
+ db.setCatalogName(extractSqlString(dbline[6]));
db.setParameters(MetaStoreUtils.trimMapNulls(dbParams,convertMapNullsToEmptyStrings));
if (LOG.isDebugEnabled()){
LOG.debug("getDatabase: directsql returning db " + db.getName()
@@ -389,20 +394,22 @@ class MetaStoreDirectSql {
/**
* Get table names by using direct SQL queries.
- *
+ * @param catName catalog name
* @param dbName Metastore database namme
* @param tableType Table type, or null if we want to get all tables
* @return list of table names
*/
- public List<String> getTables(String dbName, TableType tableType) throws MetaException {
+ public List<String> getTables(String catName, String dbName, TableType tableType)
+ throws MetaException {
String queryText = "SELECT " + TBLS + ".\"TBL_NAME\""
+ " FROM " + TBLS + " "
+ " INNER JOIN " + DBS + " ON " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" "
- + " WHERE " + DBS + ".\"NAME\" = ? "
+ + " WHERE " + DBS + ".\"NAME\" = ? AND " + DBS + ".\"CTLG_NAME\" = ? "
+ (tableType == null ? "" : "AND " + TBLS + ".\"TBL_TYPE\" = ? ") ;
- List<String> pms = new ArrayList<String>();
+ List<String> pms = new ArrayList<>();
pms.add(dbName);
+ pms.add(catName);
if (tableType != null) {
pms.add(tableType.toString());
}
@@ -436,13 +443,15 @@ class MetaStoreDirectSql {
/**
* Gets partitions by using direct SQL queries.
* Note that batching is not needed for this method - list of names implies the batch size;
+ * @param catName Metastore catalog name.
* @param dbName Metastore db name.
* @param tblName Metastore table name.
* @param partNames Partition names to get.
* @return List of partitions.
*/
- public List<Partition> getPartitionsViaSqlFilter(final String dbName, final String tblName,
- List<String> partNames) throws MetaException {
+ public List<Partition> getPartitionsViaSqlFilter(final String catName, final String dbName,
+ final String tblName, List<String> partNames)
+ throws MetaException {
if (partNames.isEmpty()) {
return Collections.emptyList();
}
@@ -450,7 +459,7 @@ class MetaStoreDirectSql {
@Override
public List<Partition> run(List<String> input) throws MetaException {
String filter = "" + PARTITIONS + ".\"PART_NAME\" in (" + makeParams(input.size()) + ")";
- return getPartitionsViaSqlFilterInternal(dbName, tblName, null, filter, input,
+ return getPartitionsViaSqlFilterInternal(catName, dbName, tblName, null, filter, input,
Collections.<String>emptyList(), null);
}
});
@@ -465,13 +474,15 @@ class MetaStoreDirectSql {
public List<Partition> getPartitionsViaSqlFilter(
SqlFilterForPushdown filter, Integer max) throws MetaException {
Boolean isViewTable = isViewTable(filter.table);
- return getPartitionsViaSqlFilterInternal(filter.table.getDbName(), filter.table.getTableName(),
- isViewTable, filter.filter, filter.params, filter.joins, max);
+ String catName = filter.table.isSetCatName() ? filter.table.getCatName() :
+ DEFAULT_CATALOG_NAME;
+ return getPartitionsViaSqlFilterInternal(catName, filter.table.getDbName(),
+ filter.table.getTableName(), isViewTable, filter.filter, filter.params, filter.joins, max);
}
public static class SqlFilterForPushdown {
- private final List<Object> params = new ArrayList<Object>();
- private final List<String> joins = new ArrayList<String>();
+ private final List<Object> params = new ArrayList<>();
+ private final List<String> joins = new ArrayList<>();
private String filter;
private Table table;
}
@@ -488,14 +499,15 @@ class MetaStoreDirectSql {
/**
* Gets all partitions of a table by using direct SQL queries.
+ * @param catName Metastore catalog name.
* @param dbName Metastore db name.
* @param tblName Metastore table name.
* @param max The maximum number of partitions to return.
* @return List of partitions.
*/
- public List<Partition> getPartitions(
+ public List<Partition> getPartitions(String catName,
String dbName, String tblName, Integer max) throws MetaException {
- return getPartitionsViaSqlFilterInternal(dbName, tblName, null,
+ return getPartitionsViaSqlFilterInternal(catName, dbName, tblName, null,
null, Collections.<String>emptyList(), Collections.<String>emptyList(), max);
}
@@ -504,13 +516,13 @@ class MetaStoreDirectSql {
t.getTableType().equals(TableType.VIRTUAL_VIEW.toString()) : null;
}
- private boolean isViewTable(String dbName, String tblName) throws MetaException {
+ private boolean isViewTable(String catName, String dbName, String tblName) throws MetaException {
Query query = null;
try {
String queryText = "select \"TBL_TYPE\" from " + TBLS + "" +
" inner join " + DBS + " on " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" " +
- " where " + TBLS + ".\"TBL_NAME\" = ? and " + DBS + ".\"NAME\" = ?";
- Object[] params = new Object[] { tblName, dbName };
+ " where " + TBLS + ".\"TBL_NAME\" = ? and " + DBS + ".\"NAME\" = ? and " + DBS + ".\"CTLG_NAME\" = ?";
+ Object[] params = new Object[] { tblName, dbName, catName };
query = pm.newQuery("javax.jdo.query.SQL", queryText);
query.setUnique(true);
Object result = executeWithArray(query, params, queryText);
@@ -536,11 +548,13 @@ class MetaStoreDirectSql {
* @param max The maximum number of partitions to return.
* @return List of partition objects.
*/
- private List<Partition> getPartitionsViaSqlFilterInternal(String dbName, String tblName,
- final Boolean isView, String sqlFilter, List<? extends Object> paramsForFilter,
- List<String> joinsForFilter, Integer max) throws MetaException {
+ private List<Partition> getPartitionsViaSqlFilterInternal(
+ String catName, String dbName, String tblName, final Boolean isView, String sqlFilter,
+ List<? extends Object> paramsForFilter, List<String> joinsForFilter,Integer max)
+ throws MetaException {
boolean doTrace = LOG.isDebugEnabled();
final String dbNameLcase = dbName.toLowerCase(), tblNameLcase = tblName.toLowerCase();
+ final String catNameLcase = normalizeSpace(catName);
// We have to be mindful of order during filtering if we are not returning all partitions.
String orderForFilter = (max != null) ? " order by \"PART_NAME\" asc" : "";
@@ -559,12 +573,14 @@ class MetaStoreDirectSql {
+ " inner join " + DBS + " on " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" "
+ " and " + DBS + ".\"NAME\" = ? "
+ join(joinsForFilter, ' ')
- + (StringUtils.isBlank(sqlFilter) ? "" : (" where " + sqlFilter)) + orderForFilter;
- Object[] params = new Object[paramsForFilter.size() + 2];
+ + " where " + DBS + ".\"CTLG_NAME\" = ? "
+ + (StringUtils.isBlank(sqlFilter) ? "" : (" and " + sqlFilter)) + orderForFilter;
+ Object[] params = new Object[paramsForFilter.size() + 3];
params[0] = tblNameLcase;
params[1] = dbNameLcase;
+ params[2] = catNameLcase;
for (int i = 0; i < paramsForFilter.size(); ++i) {
- params[i + 2] = paramsForFilter.get(i);
+ params[i + 3] = paramsForFilter.get(i);
}
long start = doTrace ? System.nanoTime() : 0;
@@ -583,7 +599,8 @@ class MetaStoreDirectSql {
List<Partition> result = runBatched(sqlResult, new Batchable<Object, Partition>() {
@Override
public List<Partition> run(List<Object> input) throws MetaException {
- return getPartitionsFromPartitionIds(dbNameLcase, tblNameLcase, isView, input);
+ return getPartitionsFromPartitionIds(catNameLcase, dbNameLcase, tblNameLcase, isView,
+ input);
}
});
@@ -592,7 +609,7 @@ class MetaStoreDirectSql {
}
/** Should be called with the list short enough to not trip up Oracle/etc. */
- private List<Partition> getPartitionsFromPartitionIds(String dbName, String tblName,
+ private List<Partition> getPartitionsFromPartitionIds(String catName, String dbName, String tblName,
Boolean isView, List<Object> partIdList) throws MetaException {
boolean doTrace = LOG.isDebugEnabled();
int idStringWidth = (int)Math.ceil(Math.log10(partIdList.size())) + 1; // 1 for comma
@@ -635,6 +652,7 @@ class MetaStoreDirectSql {
StringBuilder colsSb = new StringBuilder(7); // We expect that there's only one field schema.
tblName = tblName.toLowerCase();
dbName = dbName.toLowerCase();
+ catName = catName.toLowerCase();
for (Object[] fields : sqlResult) {
// Here comes the ugly part...
long partitionId = extractSqlLong(fields[0]);
@@ -644,7 +662,7 @@ class MetaStoreDirectSql {
// A partition must have at least sdId and serdeId set, or nothing set if it's a view.
if (sdId == null || serdeId == null) {
if (isView == null) {
- isView = isViewTable(dbName, tblName);
+ isView = isViewTable(catName, dbName, tblName);
}
if ((sdId != null || colId != null || serdeId != null) || !isView) {
throw new MetaException("Unexpected null for one of the IDs, SD " + sdId +
@@ -655,8 +673,9 @@ class MetaStoreDirectSql {
Partition part = new Partition();
orderedResult.add(part);
// Set the collection fields; some code might not check presence before accessing them.
- part.setParameters(new HashMap<String, String>());
+ part.setParameters(new HashMap<>());
part.setValues(new ArrayList<String>());
+ part.setCatName(catName);
part.setDbName(dbName);
part.setTableName(tblName);
if (fields[4] != null) part.setCreateTime(extractSqlInt(fields[4]));
@@ -910,6 +929,7 @@ class MetaStoreDirectSql {
public int getNumPartitionsViaSqlFilter(SqlFilterForPushdown filter) throws MetaException {
boolean doTrace = LOG.isDebugEnabled();
+ String catName = filter.table.getCatName().toLowerCase();
String dbName = filter.table.getDbName().toLowerCase();
String tblName = filter.table.getTableName().toLowerCase();
@@ -920,13 +940,15 @@ class MetaStoreDirectSql {
+ " inner join " + DBS + " on " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" "
+ " and " + DBS + ".\"NAME\" = ? "
+ join(filter.joins, ' ')
- + (filter.filter == null || filter.filter.trim().isEmpty() ? "" : (" where " + filter.filter));
+ + " where " + DBS + ".\"CTLG_NAME\" = ? "
+ + (filter.filter == null || filter.filter.trim().isEmpty() ? "" : (" and " + filter.filter));
- Object[] params = new Object[filter.params.size() + 2];
+ Object[] params = new Object[filter.params.size() + 3];
params[0] = tblName;
params[1] = dbName;
+ params[2] = catName;
for (int i = 0; i < filter.params.size(); ++i) {
- params[i + 2] = filter.params.get(i);
+ params[i + 3] = filter.params.get(i);
}
long start = doTrace ? System.nanoTime() : 0;
@@ -1291,10 +1313,12 @@ class MetaStoreDirectSql {
if (dbHasJoinCastBug) {
// This is a workaround for DERBY-6358 and Oracle bug; it is pretty horrible.
tableValue += (" and " + TBLS + ".\"TBL_NAME\" = ? and " + DBS + ".\"NAME\" = ? and "
+ + DBS + ".\"CTLG_NAME\" = ? and "
+ "\"FILTER" + partColIndex + "\".\"PART_ID\" = " + PARTITIONS + ".\"PART_ID\" and "
+ "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex);
params.add(table.getTableName().toLowerCase());
params.add(table.getDbName().toLowerCase());
+ params.add(table.getCatName().toLowerCase());
}
tableValue += " then " + tableValue0 + " else null end)";
}
@@ -1311,29 +1335,32 @@ class MetaStoreDirectSql {
/**
* Retrieve the column statistics for the specified columns of the table. NULL
* is returned if the columns are not provided.
+ * @param catName the catalog name of the table
* @param dbName the database name of the table
* @param tableName the table name
* @param colNames the list of the column names
* @return the column statistics for the specified columns
* @throws MetaException
*/
- public ColumnStatistics getTableStats(final String dbName, final String tableName,
- List<String> colNames, boolean enableBitVector) throws MetaException {
+ public ColumnStatistics getTableStats(final String catName, final String dbName,
+ final String tableName, List<String> colNames,
+ boolean enableBitVector) throws MetaException {
if (colNames == null || colNames.isEmpty()) {
return null;
}
final boolean doTrace = LOG.isDebugEnabled();
- final String queryText0 = "select " + getStatsList(enableBitVector) + " from " + TAB_COL_STATS + " "
- + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in (";
+ final String queryText0 = "select " + getStatsList(enableBitVector) + " from " + TAB_COL_STATS
+ + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in (";
Batchable<String, Object[]> b = new Batchable<String, Object[]>() {
@Override
public List<Object[]> run(List<String> input) throws MetaException {
String queryText = queryText0 + makeParams(input.size()) + ")";
- Object[] params = new Object[input.size() + 2];
- params[0] = dbName;
- params[1] = tableName;
+ Object[] params = new Object[input.size() + 3];
+ params[0] = catName;
+ params[1] = dbName;
+ params[2] = tableName;
for (int i = 0; i < input.size(); ++i) {
- params[i + 2] = input.get(i);
+ params[i + 3] = input.get(i);
}
long start = doTrace ? System.nanoTime() : 0;
Query query = pm.newQuery("javax.jdo.query.SQL", queryText);
@@ -1352,12 +1379,13 @@ class MetaStoreDirectSql {
return null;
}
ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tableName);
+ csd.setCatName(catName);
ColumnStatistics result = makeColumnStats(list, csd, 0);
b.closeAllQueries();
return result;
}
- public AggrStats aggrColStatsForPartitions(String dbName, String tableName,
+ public AggrStats aggrColStatsForPartitions(String catName, String dbName, String tableName,
List<String> partNames, List<String> colNames, boolean useDensityFunctionForNDVEstimation,
double ndvTuner, boolean enableBitVector) throws MetaException {
if (colNames.isEmpty() || partNames.isEmpty()) {
@@ -1379,33 +1407,33 @@ class MetaStoreDirectSql {
boolean computePartsFound = true;
for (String colName : colNames) {
// Check the cache first
- colStatsAggrCached = aggrStatsCache.get(dbName, tableName, colName, partNames);
+ colStatsAggrCached = aggrStatsCache.get(catName, dbName, tableName, colName, partNames);
if (colStatsAggrCached != null) {
colStatsList.add(colStatsAggrCached.getColStats());
partsFound = colStatsAggrCached.getNumPartsCached();
} else {
if (computePartsFound) {
- partsFound = partsFoundForPartitions(dbName, tableName, partNames, colNames);
+ partsFound = partsFoundForPartitions(catName, dbName, tableName, partNames, colNames);
computePartsFound = false;
}
- List<String> colNamesForDB = new ArrayList<String>();
+ List<String> colNamesForDB = new ArrayList<>();
colNamesForDB.add(colName);
// Read aggregated stats for one column
colStatsAggrFromDB =
- columnStatisticsObjForPartitions(dbName, tableName, partNames, colNamesForDB,
+ columnStatisticsObjForPartitions(catName, dbName, tableName, partNames, colNamesForDB,
partsFound, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector);
if (!colStatsAggrFromDB.isEmpty()) {
ColumnStatisticsObj colStatsAggr = colStatsAggrFromDB.get(0);
colStatsList.add(colStatsAggr);
// Update the cache to add this new aggregate node
- aggrStatsCache.add(dbName, tableName, colName, partsFound, colStatsAggr, bloomFilter);
+ aggrStatsCache.add(catName, dbName, tableName, colName, partsFound, colStatsAggr, bloomFilter);
}
}
}
} else {
- partsFound = partsFoundForPartitions(dbName, tableName, partNames, colNames);
+ partsFound = partsFoundForPartitions(catName, dbName, tableName, partNames, colNames);
colStatsList =
- columnStatisticsObjForPartitions(dbName, tableName, partNames, colNames, partsFound,
+ columnStatisticsObjForPartitions(catName, dbName, tableName, partNames, colNames, partsFound,
useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector);
}
LOG.info("useDensityFunctionForNDVEstimation = " + useDensityFunctionForNDVEstimation
@@ -1423,12 +1451,13 @@ class MetaStoreDirectSql {
return bloomFilter;
}
- private long partsFoundForPartitions(final String dbName, final String tableName,
+ private long partsFoundForPartitions(
+ final String catName, final String dbName, final String tableName,
final List<String> partNames, List<String> colNames) throws MetaException {
assert !colNames.isEmpty() && !partNames.isEmpty();
final boolean doTrace = LOG.isDebugEnabled();
final String queryText0 = "select count(\"COLUMN_NAME\") from " + PART_COL_STATS + ""
- + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ " and \"COLUMN_NAME\" in (%1$s) and \"PARTITION_NAME\" in (%2$s)"
+ " group by \"PARTITION_NAME\"";
List<Long> allCounts = runBatched(colNames, new Batchable<String, Long>() {
@@ -1444,7 +1473,7 @@ class MetaStoreDirectSql {
Query query = pm.newQuery("javax.jdo.query.SQL", queryText);
try {
Object qResult = executeWithArray(query, prepareParams(
- dbName, tableName, inputPartNames, inputColName), queryText);
+ catName, dbName, tableName, inputPartNames, inputColName), queryText);
long end = doTrace ? System.nanoTime() : 0;
timingTrace(doTrace, queryText, start, end);
ForwardQueryResult<?> fqr = (ForwardQueryResult<?>) qResult;
@@ -1469,7 +1498,8 @@ class MetaStoreDirectSql {
return partsFound;
}
- private List<ColumnStatisticsObj> columnStatisticsObjForPartitions(final String dbName,
+ private List<ColumnStatisticsObj> columnStatisticsObjForPartitions(
+ final String catName, final String dbName,
final String tableName, final List<String> partNames, List<String> colNames, long partsFound,
final boolean useDensityFunctionForNDVEstimation, final double ndvTuner, final boolean enableBitVector) throws MetaException {
final boolean areAllPartsFound = (partsFound == partNames.size());
@@ -1479,7 +1509,7 @@ class MetaStoreDirectSql {
return runBatched(partNames, new Batchable<String, ColumnStatisticsObj>() {
@Override
public List<ColumnStatisticsObj> run(List<String> inputPartNames) throws MetaException {
- return columnStatisticsObjForPartitionsBatch(dbName, tableName, inputPartNames,
+ return columnStatisticsObjForPartitionsBatch(catName, dbName, tableName, inputPartNames,
inputColNames, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector);
}
});
@@ -1487,10 +1517,10 @@ class MetaStoreDirectSql {
});
}
- public List<ColStatsObjWithSourceInfo> getColStatsForAllTablePartitions(String dbName,
+ public List<ColStatsObjWithSourceInfo> getColStatsForAllTablePartitions(String catName, String dbName,
boolean enableBitVector) throws MetaException {
String queryText = "select \"TABLE_NAME\", \"PARTITION_NAME\", " + getStatsList(enableBitVector)
- + " from " + " " + PART_COL_STATS + " where \"DB_NAME\" = ?";
+ + " from " + " " + PART_COL_STATS + " where \"DB_NAME\" = ? and \"CAT_NAME\" = ?";
long start = 0;
long end = 0;
Query query = null;
@@ -1500,7 +1530,7 @@ class MetaStoreDirectSql {
List<ColStatsObjWithSourceInfo> colStatsForDB = new ArrayList<ColStatsObjWithSourceInfo>();
try {
query = pm.newQuery("javax.jdo.query.SQL", queryText);
- qResult = executeWithArray(query, new Object[] { dbName }, queryText);
+ qResult = executeWithArray(query, new Object[] { dbName, catName }, queryText);
if (qResult == null) {
query.closeAll();
return colStatsForDB;
@@ -1512,7 +1542,7 @@ class MetaStoreDirectSql {
String tblName = (String) row[0];
String partName = (String) row[1];
ColumnStatisticsObj colStatObj = prepareCSObj(row, 2);
- colStatsForDB.add(new ColStatsObjWithSourceInfo(colStatObj, dbName, tblName, partName));
+ colStatsForDB.add(new ColStatsObjWithSourceInfo(colStatObj, catName, dbName, tblName, partName));
Deadline.checkTimeout();
}
} finally {
@@ -1522,31 +1552,31 @@ class MetaStoreDirectSql {
}
/** Should be called with the list short enough to not trip up Oracle/etc. */
- private List<ColumnStatisticsObj> columnStatisticsObjForPartitionsBatch(String dbName,
+ private List<ColumnStatisticsObj> columnStatisticsObjForPartitionsBatch(String catName, String dbName,
String tableName, List<String> partNames, List<String> colNames, boolean areAllPartsFound,
boolean useDensityFunctionForNDVEstimation, double ndvTuner, boolean enableBitVector)
throws MetaException {
if (enableBitVector) {
- return aggrStatsUseJava(dbName, tableName, partNames, colNames, areAllPartsFound,
+ return aggrStatsUseJava(catName, dbName, tableName, partNames, colNames, areAllPartsFound,
useDensityFunctionForNDVEstimation, ndvTuner);
} else {
- return aggrStatsUseDB(dbName, tableName, partNames, colNames, areAllPartsFound,
+ return aggrStatsUseDB(catName, dbName, tableName, partNames, colNames, areAllPartsFound,
useDensityFunctionForNDVEstimation, ndvTuner);
}
}
- private List<ColumnStatisticsObj> aggrStatsUseJava(String dbName, String tableName,
+ private List<ColumnStatisticsObj> aggrStatsUseJava(String catName, String dbName, String tableName,
List<String> partNames, List<String> colNames, boolean areAllPartsFound,
boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException {
// 1. get all the stats for colNames in partNames;
List<ColumnStatistics> partStats =
- getPartitionStats(dbName, tableName, partNames, colNames, true);
+ getPartitionStats(catName, dbName, tableName, partNames, colNames, true);
// 2. use util function to aggr stats
- return MetaStoreUtils.aggrPartitionStats(partStats, dbName, tableName, partNames, colNames,
+ return MetaStoreUtils.aggrPartitionStats(partStats, catName, dbName, tableName, partNames, colNames,
areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner);
}
- private List<ColumnStatisticsObj> aggrStatsUseDB(String dbName,
+ private List<ColumnStatisticsObj> aggrStatsUseDB(String catName, String dbName,
String tableName, List<String> partNames, List<String> colNames, boolean areAllPartsFound,
boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException {
// TODO: all the extrapolation logic should be moved out of this class,
@@ -1573,7 +1603,7 @@ class MetaStoreDirectSql {
+ "avg((\"DOUBLE_HIGH_VALUE\"-\"DOUBLE_LOW_VALUE\")/\"NUM_DISTINCTS\"),"
+ "avg((cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)-cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal))/\"NUM_DISTINCTS\"),"
+ "sum(\"NUM_DISTINCTS\")" + " from " + PART_COL_STATS + ""
- + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? ";
+ + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? ";
String queryText = null;
long start = 0;
long end = 0;
@@ -1589,7 +1619,7 @@ class MetaStoreDirectSql {
+ " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
start = doTrace ? System.nanoTime() : 0;
query = pm.newQuery("javax.jdo.query.SQL", queryText);
- qResult = executeWithArray(query, prepareParams(dbName, tableName, partNames, colNames),
+ qResult = executeWithArray(query, prepareParams(catName, dbName, tableName, partNames, colNames),
queryText);
if (qResult == null) {
query.closeAll();
@@ -1612,13 +1642,13 @@ class MetaStoreDirectSql {
List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(colNames.size());
queryText = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", count(\"PARTITION_NAME\") "
+ " from " + PART_COL_STATS
- + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"
+ " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
start = doTrace ? System.nanoTime() : 0;
query = pm.newQuery("javax.jdo.query.SQL", queryText);
- qResult = executeWithArray(query, prepareParams(dbName, tableName, partNames, colNames),
+ qResult = executeWithArray(query, prepareParams(catName, dbName, tableName, partNames, colNames),
queryText);
end = doTrace ? System.nanoTime() : 0;
timingTrace(doTrace, queryText, start, end);
@@ -1653,7 +1683,7 @@ class MetaStoreDirectSql {
start = doTrace ? System.nanoTime() : 0;
query = pm.newQuery("javax.jdo.query.SQL", queryText);
qResult = executeWithArray(query,
- prepareParams(dbName, tableName, partNames, noExtraColumnNames), queryText);
+ prepareParams(catName, dbName, tableName, partNames, noExtraColumnNames), queryText);
if (qResult == null) {
query.closeAll();
return Collections.emptyList();
@@ -1677,7 +1707,7 @@ class MetaStoreDirectSql {
// get sum for all columns to reduce the number of queries
Map<String, Map<Integer, Object>> sumMap = new HashMap<String, Map<Integer, Object>>();
queryText = "select \"COLUMN_NAME\", sum(\"NUM_NULLS\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\"), sum(\"NUM_DISTINCTS\")"
- + " from " + PART_COL_STATS + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ + " from " + PART_COL_STATS + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ " and \"COLUMN_NAME\" in (" + makeParams(extraColumnNameTypeParts.size())
+ ") and \"PARTITION_NAME\" in (" + makeParams(partNames.size())
+ ") group by \"COLUMN_NAME\"";
@@ -1686,7 +1716,7 @@ class MetaStoreDirectSql {
List<String> extraColumnNames = new ArrayList<String>();
extraColumnNames.addAll(extraColumnNameTypeParts.keySet());
qResult = executeWithArray(query,
- prepareParams(dbName, tableName, partNames, extraColumnNames), queryText);
+ prepareParams(catName, dbName, tableName, partNames, extraColumnNames), queryText);
if (qResult == null) {
query.closeAll();
return Collections.emptyList();
@@ -1750,20 +1780,20 @@ class MetaStoreDirectSql {
if (!decimal) {
queryText = "select \"" + colStatName
+ "\",\"PARTITION_NAME\" from " + PART_COL_STATS
- + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?"
+ + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?"
+ " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ " order by \"" + colStatName + "\"";
} else {
queryText = "select \"" + colStatName
+ "\",\"PARTITION_NAME\" from " + PART_COL_STATS
- + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?"
+ + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?"
+ " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ " order by cast(\"" + colStatName + "\" as decimal)";
}
start = doTrace ? System.nanoTime() : 0;
query = pm.newQuery("javax.jdo.query.SQL", queryText);
qResult = executeWithArray(query,
- prepareParams(dbName, tableName, partNames, Arrays.asList(colName)), queryText);
+ prepareParams(catName, dbName, tableName, partNames, Arrays.asList(colName)), queryText);
if (qResult == null) {
query.closeAll();
return Collections.emptyList();
@@ -1786,13 +1816,13 @@ class MetaStoreDirectSql {
+ "avg((\"LONG_HIGH_VALUE\"-\"LONG_LOW_VALUE\")/cast(\"NUM_DISTINCTS\" as decimal)),"
+ "avg((\"DOUBLE_HIGH_VALUE\"-\"DOUBLE_LOW_VALUE\")/\"NUM_DISTINCTS\"),"
+ "avg((cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)-cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal))/\"NUM_DISTINCTS\")"
- + " from " + PART_COL_STATS + "" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?"
+ + " from " + PART_COL_STATS + "" + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ?"
+ " and \"COLUMN_NAME\" = ?" + " and \"PARTITION_NAME\" in ("
+ makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\"";
start = doTrace ? System.nanoTime() : 0;
query = pm.newQuery("javax.jdo.query.SQL", queryText);
qResult = executeWithArray(query,
- prepareParams(dbName, tableName, partNames, Arrays.asList(colName)), queryText);
+ prepareParams(catName, dbName, tableName, partNames, Arrays.asList(colName)), queryText);
if (qResult == null) {
query.closeAll();
return Collections.emptyList();
@@ -1837,11 +1867,12 @@ class MetaStoreDirectSql {
return cso;
}
- private Object[] prepareParams(String dbName, String tableName, List<String> partNames,
- List<String> colNames) throws MetaException {
+ private Object[] prepareParams(String catName, String dbName, String tableName,
+ List<String> partNames, List<String> colNames) throws MetaException {
- Object[] params = new Object[colNames.size() + partNames.size() + 2];
+ Object[] params = new Object[colNames.size() + partNames.size() + 3];
int paramI = 0;
+ params[paramI++] = catName;
params[paramI++] = dbName;
params[paramI++] = tableName;
for (String colName : colNames) {
@@ -1854,14 +1885,16 @@ class MetaStoreDirectSql {
return params;
}
- public List<ColumnStatistics> getPartitionStats(final String dbName, final String tableName,
- final List<String> partNames, List<String> colNames, boolean enableBitVector) throws MetaException {
+ public List<ColumnStatistics> getPartitionStats(
+ final String catName, final String dbName, final String tableName, final List<String> partNames,
+ List<String> colNames, boolean enableBitVector) throws MetaException {
if (colNames.isEmpty() || partNames.isEmpty()) {
return Collections.emptyList();
}
final boolean doTrace = LOG.isDebugEnabled();
final String queryText0 = "select \"PARTITION_NAME\", " + getStatsList(enableBitVector) + " from "
- + " " + PART_COL_STATS + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\""
+ + " " + PART_COL_STATS + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and " +
+ "\"COLUMN_NAME\""
+ " in (%1$s) AND \"PARTITION_NAME\" in (%2$s) order by \"PARTITION_NAME\"";
Batchable<String, Object[]> b = new Batchable<String, Object[]>() {
@Override
@@ -1874,7 +1907,7 @@ class MetaStoreDirectSql {
long start = doTrace ? System.nanoTime() : 0;
Query query = pm.newQuery("javax.jdo.query.SQL", queryText);
Object qResult = executeWithArray(query, prepareParams(
- dbName, tableName, inputPartNames, inputColNames), queryText);
+ catName, dbName, tableName, inputPartNames, inputColNames), queryText);
timingTrace(doTrace, queryText0, start, (doTrace ? System.nanoTime() : 0));
if (qResult == null) {
query.closeAll();
@@ -1904,6 +1937,7 @@ class MetaStoreDirectSql {
continue;
} else if (from != i) {
ColumnStatisticsDesc csd = new ColumnStatisticsDesc(false, dbName, tableName);
+ csd.setCatName(catName);
csd.setPartName(lastPartName);
result.add(makeColumnStats(list.subList(from, i), csd, 1));
}
@@ -2036,8 +2070,10 @@ class MetaStoreDirectSql {
return result;
}
- public List<SQLForeignKey> getForeignKeys(String parent_db_name, String parent_tbl_name, String foreign_db_name, String foreign_tbl_name) throws MetaException {
- List<SQLForeignKey> ret = new ArrayList<SQLForeignKey>();
+ public List<SQLForeignKey> getForeignKeys(String catName, String parent_db_name,
+ String parent_tbl_name, String foreign_db_name,
+ String foreign_tbl_name) throws MetaException {
+ List<SQLForeignKey> ret = new ArrayList<>();
String queryText =
"SELECT \"D2\".\"NAME\", \"T2\".\"TBL_NAME\", "
+ "CASE WHEN \"C2\".\"COLUMN_NAME\" IS NOT NULL THEN \"C2\".\"COLUMN_NAME\" "
@@ -2065,6 +2101,7 @@ class MetaStoreDirectSql {
+ " \"P2\".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" "
+ " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = " + MConstraint.FOREIGN_KEY_CONSTRAINT
+ " AND \"KEY_CONSTRAINTS2\".\"CONSTRAINT_TYPE\" = " + MConstraint.PRIMARY_KEY_CONSTRAINT + " AND"
+ + " " + DBS + ".\"CTLG_NAME\" = ? AND"
+ (foreign_db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND")
+ (foreign_tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? AND")
+ (parent_tbl_name == null ? "" : " \"T2\".\"TBL_NAME\" = ? AND")
@@ -2075,6 +2112,7 @@ class MetaStoreDirectSql {
queryText = queryText.substring(0, queryText.length()-3);
}
List<String> pms = new ArrayList<String>();
+ pms.add(catName);
if (foreign_db_name != null) {
pms.add(foreign_db_name);
}
@@ -2114,19 +2152,22 @@ class MetaStoreDirectSql {
validate,
rely
);
- ret.add(currKey);
+ currKey.setCatName(catName);
+ ret.add(currKey);
}
}
return ret;
}
- public List<SQLPrimaryKey> getPrimaryKeys(String db_name, String tbl_name) throws MetaException {
- List<SQLPrimaryKey> ret = new ArrayList<SQLPrimaryKey>();
+ public List<SQLPrimaryKey> getPrimaryKeys(String catName, String db_name, String tbl_name)
+ throws MetaException {
+ List<SQLPrimaryKey> ret = new ArrayList<>();
String queryText =
"SELECT " + DBS + ".\"NAME\", " + TBLS + ".\"TBL_NAME\", "
+ "CASE WHEN " + COLUMNS_V2 + ".\"COLUMN_NAME\" IS NOT NULL THEN " + COLUMNS_V2 + ".\"COLUMN_NAME\" "
+ "ELSE " + PARTITION_KEYS + ".\"PKEY_NAME\" END, " + KEY_CONSTRAINTS + ".\"POSITION\", "
- + "" + KEY_CONSTRAINTS + ".\"CONSTRAINT_NAME\", " + KEY_CONSTRAINTS + ".\"ENABLE_VALIDATE_RELY\" "
+ + KEY_CONSTRAINTS + ".\"CONSTRAINT_NAME\", " + KEY_CONSTRAINTS + ".\"ENABLE_VALIDATE_RELY\", "
+ + DBS + ".\"CTLG_NAME\""
+ " from " + TBLS + " "
+ " INNER JOIN " + KEY_CONSTRAINTS + " ON " + TBLS + ".\"TBL_ID\" = " + KEY_CONSTRAINTS + ".\"PARENT_TBL_ID\" "
+ " INNER JOIN " + DBS + " ON " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" "
@@ -2135,6 +2176,7 @@ class MetaStoreDirectSql {
+ " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND "
+ " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" "
+ " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.PRIMARY_KEY_CONSTRAINT + " AND"
+ + " " + DBS + ".\"CTLG_NAME\" = ? AND"
+ (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND")
+ (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ;
@@ -2142,7 +2184,8 @@ class MetaStoreDirectSql {
if (queryText.endsWith("AND")) {
queryText = queryText.substring(0, queryText.length()-3);
}
- List<String> pms = new ArrayList<String>();
+ List<String> pms = new ArrayList<>();
+ pms.add(catName);
if (db_name != null) {
pms.add(db_name);
}
@@ -2168,13 +2211,14 @@ class MetaStoreDirectSql {
enable,
validate,
rely);
- ret.add(currKey);
+ currKey.setCatName(extractSqlString(line[6]));
+ ret.add(currKey);
}
}
return ret;
}
- public List<SQLUniqueConstraint> getUniqueConstraints(String db_name, String tbl_name)
+ public List<SQLUniqueConstraint> getUniqueConstraints(String catName, String db_name, String tbl_name)
throws MetaException {
List<SQLUniqueConstraint> ret = new ArrayList<SQLUniqueConstraint>();
String queryText =
@@ -2190,6 +2234,7 @@ class MetaStoreDirectSql {
+ " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND "
+ " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" "
+ " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.UNIQUE_CONSTRAINT + " AND"
+ + " " + DBS + ".\"CTLG_NAME\" = ? AND"
+ (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND")
+ (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ;
@@ -2198,6 +2243,7 @@ class MetaStoreDirectSql {
queryText = queryText.substring(0, queryText.length()-3);
}
List<String> pms = new ArrayList<String>();
+ pms.add(catName);
if (db_name != null) {
pms.add(db_name);
}
@@ -2215,23 +2261,23 @@ class MetaStoreDirectSql {
boolean enable = (enableValidateRely & 4) != 0;
boolean validate = (enableValidateRely & 2) != 0;
boolean rely = (enableValidateRely & 1) != 0;
- SQLUniqueConstraint currConstraint = new SQLUniqueConstraint(
- extractSqlString(line[0]),
- extractSqlString(line[1]),
- extractSqlString(line[2]),
- extractSqlInt(line[3]), extractSqlString(line[4]),
- enable,
- validate,
- rely);
- ret.add(currConstraint);
+ ret.add(new SQLUniqueConstraint(
+ catName,
+ extractSqlString(line[0]),
+ extractSqlString(line[1]),
+ extractSqlString(line[2]),
+ extractSqlInt(line[3]), extractSqlString(line[4]),
+ enable,
+ validate,
+ rely));
}
}
return ret;
}
- public List<SQLNotNullConstraint> getNotNullConstraints(String db_name, String tbl_name)
+ public List<SQLNotNullConstraint> getNotNullConstraints(String catName, String db_name, String tbl_name)
throws MetaException {
- List<SQLNotNullConstraint> ret = new ArrayList<SQLNotNullConstraint>();
+ List<SQLNotNullConstraint> ret = new ArrayList<>();
String queryText =
"SELECT " + DBS + ".\"NAME\", " + TBLS + ".\"TBL_NAME\","
+ "CASE WHEN " + COLUMNS_V2 + ".\"COLUMN_NAME\" IS NOT NULL THEN " + COLUMNS_V2 + ".\"COLUMN_NAME\" "
@@ -2245,6 +2291,7 @@ class MetaStoreDirectSql {
+ " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND "
+ " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" "
+ " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.NOT_NULL_CONSTRAINT + " AND"
+ + " " + DBS + ".\"CTLG_NAME\" = ? AND"
+ (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND")
+ (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ;
@@ -2252,7 +2299,8 @@ class MetaStoreDirectSql {
if (queryText.endsWith("AND")) {
queryText = queryText.substring(0, queryText.length()-3);
}
- List<String> pms = new ArrayList<String>();
+ List<String> pms = new ArrayList<>();
+ pms.add(catName);
if (db_name != null) {
pms.add(db_name);
}
@@ -2270,21 +2318,21 @@ class MetaStoreDirectSql {
boolean enable = (enableValidateRely & 4) != 0;
boolean validate = (enableValidateRely & 2) != 0;
boolean rely = (enableValidateRely & 1) != 0;
- SQLNotNullConstraint currConstraint = new SQLNotNullConstraint(
- extractSqlString(line[0]),
- extractSqlString(line[1]),
- extractSqlString(line[2]),
- extractSqlString(line[3]),
- enable,
- validate,
- rely);
- ret.add(currConstraint);
+ ret.add(new SQLNotNullConstraint(
+ catName,
+ extractSqlString(line[0]),
+ extractSqlString(line[1]),
+ extractSqlString(line[2]),
+ extractSqlString(line[3]),
+ enable,
+ validate,
+ rely));
}
}
return ret;
}
- public List<SQLDefaultConstraint> getDefaultConstraints(String db_name, String tbl_name)
+ public List<SQLDefaultConstraint> getDefaultConstraints(String catName, String db_name, String tbl_name)
throws MetaException {
List<SQLDefaultConstraint> ret = new ArrayList<SQLDefaultConstraint>();
String queryText =
@@ -2301,6 +2349,7 @@ class MetaStoreDirectSql {
+ " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND "
+ " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" "
+ " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.DEFAULT_CONSTRAINT+ " AND"
+ + " " + DBS + ".\"CTLG_NAME\" = ? AND"
+ (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND")
+ (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ;
@@ -2311,7 +2360,8 @@ class MetaStoreDirectSql {
if (LOG.isDebugEnabled()){
LOG.debug("getDefaultConstraints: directsql : " + queryText);
}
- List<String> pms = new ArrayList<String>();
+ List<String> pms = new ArrayList<>();
+ pms.add(catName);
if (db_name != null) {
pms.add(db_name);
}
@@ -2330,6 +2380,7 @@ class MetaStoreDirectSql {
boolean validate = (enableValidateRely & 2) != 0;
boolean rely = (enableValidateRely & 1) != 0;
SQLDefaultConstraint currConstraint = new SQLDefaultConstraint(
+ catName,
extractSqlString(line[0]),
extractSqlString(line[1]),
extractSqlString(line[2]),
@@ -2344,7 +2395,7 @@ class MetaStoreDirectSql {
return ret;
}
- public List<SQLCheckConstraint> getCheckConstraints(String db_name, String tbl_name)
+ public List<SQLCheckConstraint> getCheckConstraints(String catName, String db_name, String tbl_name)
throws MetaException {
List<SQLCheckConstraint> ret = new ArrayList<SQLCheckConstraint>();
String queryText =
@@ -2361,6 +2412,7 @@ class MetaStoreDirectSql {
+ " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND "
+ " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" "
+ " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.CHECK_CONSTRAINT+ " AND"
+ + " " + DBS + ".\"CTLG_NAME\" = ? AND"
+ (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND")
+ (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ;
@@ -2371,7 +2423,8 @@ class MetaStoreDirectSql {
if (LOG.isDebugEnabled()){
LOG.debug("getCheckConstraints: directsql : " + queryText);
}
- List<String> pms = new ArrayList<String>();
+ List<String> pms = new ArrayList<>();
+ pms.add(catName);
if (db_name != null) {
pms.add(db_name);
}
@@ -2390,6 +2443,7 @@ class MetaStoreDirectSql {
boolean validate = (enableValidateRely & 2) != 0;
boolean rely = (enableValidateRely & 1) != 0;
SQLCheckConstraint currConstraint = new SQLCheckConstraint(
+ catName,
extractSqlString(line[0]),
extractSqlString(line[1]),
extractSqlString(line[2]),
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
index 67600e1..569fff0 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java
@@ -35,10 +35,12 @@ import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterSchemaVersionEvent;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent;
+import org.apache.hadoop.hive.metastore.events.CreateCatalogEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent;
import org.apache.hadoop.hive.metastore.events.CreateISchemaEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropCatalogEvent;
import org.apache.hadoop.hive.metastore.events.DropConstraintEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropFunctionEvent;
@@ -222,6 +224,12 @@ public abstract class MetaStoreEventListener implements Configurable {
throws MetaException {
}
+ public void onCreateCatalog(CreateCatalogEvent createCatalogEvent) throws MetaException {
+ }
+
+ public void onDropCatalog(DropCatalogEvent dropCatalogEvent) throws MetaException {
+ }
+
@Override
public Configuration getConf() {
return this.conf;
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
index 8522afe..f7a0cd0 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java
@@ -22,12 +22,14 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.api.Catalog;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PartitionSpec;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
/**
* Metadata filter hook for metastore client. This will be useful for authorization
@@ -39,11 +41,31 @@ import org.apache.hadoop.hive.metastore.api.Table;
public interface MetaStoreFilterHook {
/**
+ * Filter a catalog object. Default implementation returns the passed in catalog.
+ * @param catalog catalog to filter
+ * @return filtered catalog
+ * @throws MetaException something bad happened
+ */
+ default Catalog filterCatalog(Catalog catalog) throws MetaException {
+ return catalog;
+ }
+
+ /**
+ * Filter a list of catalog names. Default implementation returns the passed in list.
+ * @param catalogs list of catalog names.
+ * @return filtered list of catalog names.
+ * @throws MetaException something bad happened.
+ */
+ default List<String> filterCatalogs(List<String> catalogs) throws MetaException {
+ return catalogs;
+ }
+
+ /**
* Filter given list of databases
* @param dbList
* @return List of filtered Db names
*/
- public List<String> filterDatabases(List<String> dbList) throws MetaException;
+ List<String> filterDatabases(List<String> dbList) throws MetaException;
/**
* filter to given database object if applicable
@@ -51,15 +73,27 @@ public interface MetaStoreFilterHook {
* @return the same database if it's not filtered out
* @throws NoSuchObjectException
*/
- public Database filterDatabase(Database dataBase) throws MetaException, NoSuchObjectException;
+ Database filterDatabase(Database dataBase) throws MetaException, NoSuchObjectException;
/**
* Filter given list of tables
- * @param dbName
- * @param tableList
+ * @param catName catalog name
+ * @param dbName database name
+ * @param tableList list of table returned by the metastore
* @return List of filtered table names
*/
- public List<String> filterTableNames(String dbName, List<String> tableList) throws MetaException;
+ List<String> filterTableNames(String catName, String dbName, List<String> tableList)
+ throws MetaException;
+
+ // Previously this was handled by filterTableNames. But it can't be anymore because we can no
+ // longer depend on a 1-1 mapping between table name and entry in the list.
+ /**
+ * Filter a list of TableMeta objects.
+ * @param tableMetas list of TableMetas to filter
+ * @return filtered table metas
+ * @throws MetaException something went wrong
+ */
+ List<TableMeta> filterTableMetas(List<TableMeta> tableMetas) throws MetaException;
/**
* filter to given table object if applicable
@@ -67,28 +101,28 @@ public interface MetaStoreFilterHook {
* @return the same table if it's not filtered out
* @throws NoSuchObjectException
*/
- public Table filterTable(Table table) throws MetaException, NoSuchObjectException;
+ Table filterTable(Table table) throws MetaException, NoSuchObjectException;
/**
* Filter given list of tables
* @param tableList
* @return List of filtered table names
*/
- public List<Table> filterTables(List<Table> tableList) throws MetaException;
+ List<Table> filterTables(List<Table> tableList) throws MetaException;
/**
* Filter given list of partitions
* @param partitionList
* @return
*/
- public List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException;
+ List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException;
/**
* Filter given list of partition specs
* @param partitionSpecList
* @return
*/
- public List<PartitionSpec> filterPartitionSpecs(List<PartitionSpec> partitionSpecList)
+ List<PartitionSpec> filterPartitionSpecs(List<PartitionSpec> partitionSpecList)
throws MetaException;
/**
@@ -97,18 +131,17 @@ public interface MetaStoreFilterHook {
* @return the same partition object if it's not filtered out
* @throws NoSuchObjectException
*/
- public Partition filterPartition(Partition partition) throws MetaException, NoSuchObjectException;
+ Partition filterPartition(Partition partition) throws MetaException, NoSuchObjectException;
/**
* Filter given list of partition names
- * @param dbName
- * @param tblName
- * @param partitionNames
- * @return
+ * @param catName catalog name.
+ * @param dbName database name.
+ * @param tblName table name.
+ * @param partitionNames list of partition names.
+ * @return list of filtered partition names.
*/
- public List<String> filterPartitionNames(String dbName, String tblName,
+ List<String> filterPartitionNames(String catName, String dbName, String tblName,
List<String> partitionNames) throws MetaException;
-
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java
index f5a91b4..988fca6 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java
@@ -35,10 +35,12 @@ import org.apache.hadoop.hive.metastore.events.AlterISchemaEvent;
import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterSchemaVersionEvent;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
+import org.apache.hadoop.hive.metastore.events.CreateCatalogEvent;
import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent;
import org.apache.hadoop.hive.metastore.events.CreateISchemaEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
+import org.apache.hadoop.hive.metastore.events.DropCatalogEvent;
import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.DropFunctionEvent;
import org.apache.hadoop.hive.metastore.events.DropISchemaEvent;
@@ -200,6 +202,10 @@ public class MetaStoreListenerNotifier {
listener.onDropSchemaVersion((DropSchemaVersionEvent) event);
}
})
+ .put(EventType.CREATE_CATALOG,
+ (listener, event) -> listener.onCreateCatalog((CreateCatalogEvent)event))
+ .put(EventType.DROP_CATALOG,
+ (listener, event) -> listener.onDropCatalog((DropCatalogEvent)event))
.build()
);