You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2018/03/31 01:25:34 UTC
[15/44] hive git commit: HIVE-18755 Modifications to the metastore
for catalogs (Alan Gates, reviewed by Thejas Nair)
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index de94172..3cea4b2 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.hive.metastore;
import static org.apache.commons.lang.StringUtils.join;
+import static org.apache.hadoop.hive.metastore.Warehouse.getCatalogQualifiedDbName;
+import static org.apache.hadoop.hive.metastore.Warehouse.getCatalogQualifiedTableName;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
import java.io.IOException;
@@ -69,6 +72,7 @@ import javax.sql.DataSource;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -79,6 +83,8 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.metastore.MetaStoreDirectSql.SqlFilterForPushdown;
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.BasicTxnInfo;
+import org.apache.hadoop.hive.metastore.api.Catalog;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
@@ -156,6 +162,7 @@ import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider;
import org.apache.hadoop.hive.metastore.datasource.DataSourceProviderFactory;
import org.apache.hadoop.hive.metastore.metrics.Metrics;
import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
+import org.apache.hadoop.hive.metastore.model.MCatalog;
import org.apache.hadoop.hive.metastore.model.MColumnDescriptor;
import org.apache.hadoop.hive.metastore.model.MConstraint;
import org.apache.hadoop.hive.metastore.model.MCreationMetadata;
@@ -203,6 +210,7 @@ import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.ObjectPair;
+import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.datanucleus.AbstractNucleusContext;
import org.datanucleus.ClassLoaderResolver;
@@ -792,9 +800,127 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
+ public void createCatalog(Catalog cat) throws MetaException {
+ LOG.debug("Creating catalog " + cat.getName());
+ boolean committed = false;
+ MCatalog mCat = catToMCat(cat);
+ try {
+ openTransaction();
+ pm.makePersistent(mCat);
+ committed = commitTransaction();
+ } finally {
+ if (!committed) rollbackTransaction();
+ }
+ }
+
+ @Override
+ public void alterCatalog(String catName, Catalog cat)
+ throws MetaException, InvalidOperationException {
+ if (!cat.getName().equals(catName)) {
+ throw new InvalidOperationException("You cannot change a catalog's name");
+ }
+ boolean committed = false;
+ try {
+ MCatalog mCat = getMCatalog(catName);
+ if (org.apache.commons.lang.StringUtils.isNotBlank(cat.getLocationUri())) {
+ mCat.setLocationUri(cat.getLocationUri());
+ }
+ if (org.apache.commons.lang.StringUtils.isNotBlank(cat.getDescription())) {
+ mCat.setDescription(cat.getDescription());
+ }
+ openTransaction();
+ pm.makePersistent(mCat);
+ committed = commitTransaction();
+ } finally {
+ if (!committed) rollbackTransaction();
+ }
+ }
+
+ @Override
+ public Catalog getCatalog(String catalogName) throws NoSuchObjectException, MetaException {
+ LOG.debug("Fetching catalog " + catalogName);
+ MCatalog mCat = getMCatalog(catalogName);
+ if (mCat == null) throw new NoSuchObjectException("No catalog " + catalogName);
+ return mCatToCat(mCat);
+ }
+
+ @Override
+ public List<String> getCatalogs() throws MetaException {
+ LOG.debug("Fetching all catalog names");
+ boolean commited = false;
+ List<String> catalogs = null;
+
+ String queryStr = "select name from org.apache.hadoop.hive.metastore.model.MCatalog";
+ Query query = null;
+
+ openTransaction();
+ try {
+ query = pm.newQuery(queryStr);
+ query.setResult("name");
+ catalogs = new ArrayList<>((Collection<String>) query.execute());
+ commited = commitTransaction();
+ } finally {
+ rollbackAndCleanup(commited, query);
+ }
+ Collections.sort(catalogs);
+ return catalogs;
+ }
+
+ @Override
+ public void dropCatalog(String catalogName) throws NoSuchObjectException, MetaException {
+ LOG.debug("Dropping catalog " + catalogName);
+ boolean committed = false;
+ try {
+ openTransaction();
+ MCatalog mCat = getMCatalog(catalogName);
+ pm.retrieve(mCat);
+ if (mCat == null) throw new NoSuchObjectException("No catalog " + catalogName);
+ pm.deletePersistent(mCat);
+ committed = commitTransaction();
+ } finally {
+ if (!committed) rollbackTransaction();
+ }
+ }
+
+ private MCatalog getMCatalog(String catalogName) throws MetaException {
+ boolean committed = false;
+ Query query = null;
+ try {
+ openTransaction();
+ catalogName = normalizeIdentifier(catalogName);
+ query = pm.newQuery(MCatalog.class, "name == catname");
+ query.declareParameters("java.lang.String catname");
+ query.setUnique(true);
+ MCatalog mCat = (MCatalog)query.execute(catalogName);
+ pm.retrieve(mCat);
+ committed = commitTransaction();
+ return mCat;
+ } finally {
+ rollbackAndCleanup(committed, query);
+ }
+ }
+
+ private MCatalog catToMCat(Catalog cat) {
+ MCatalog mCat = new MCatalog();
+ mCat.setName(normalizeIdentifier(cat.getName()));
+ if (cat.isSetDescription()) mCat.setDescription(cat.getDescription());
+ mCat.setLocationUri(cat.getLocationUri());
+ return mCat;
+ }
+
+ private Catalog mCatToCat(MCatalog mCat) {
+ Catalog cat = new Catalog(mCat.getName(), mCat.getLocationUri());
+ if (mCat.getDescription() != null) cat.setDescription(mCat.getDescription());
+ return cat;
+ }
+
+ @Override
public void createDatabase(Database db) throws InvalidObjectException, MetaException {
boolean commited = false;
MDatabase mdb = new MDatabase();
+ assert db.getCatalogName() != null;
+ mdb.setCatalogName(normalizeIdentifier(db.getCatalogName()));
+ assert mdb.getCatalogName() != null;
mdb.setName(db.getName().toLowerCase());
mdb.setLocationUri(db.getLocationUri());
mdb.setDescription(db.getDescription());
@@ -814,34 +940,35 @@ public class ObjectStore implements RawStore, Configurable {
}
@SuppressWarnings("nls")
- private MDatabase getMDatabase(String name) throws NoSuchObjectException {
+ private MDatabase getMDatabase(String catName, String name) throws NoSuchObjectException {
MDatabase mdb = null;
boolean commited = false;
Query query = null;
try {
openTransaction();
name = normalizeIdentifier(name);
- query = pm.newQuery(MDatabase.class, "name == dbname");
- query.declareParameters("java.lang.String dbname");
+ catName = normalizeIdentifier(catName);
+ query = pm.newQuery(MDatabase.class, "name == dbname && catalogName == catname");
+ query.declareParameters("java.lang.String dbname, java.lang.String catname");
query.setUnique(true);
- mdb = (MDatabase) query.execute(name);
+ mdb = (MDatabase) query.execute(name, catName);
pm.retrieve(mdb);
commited = commitTransaction();
} finally {
rollbackAndCleanup(commited, query);
}
if (mdb == null) {
- throw new NoSuchObjectException("There is no database named " + name);
+ throw new NoSuchObjectException("There is no database " + catName + "." + name);
}
return mdb;
}
@Override
- public Database getDatabase(String name) throws NoSuchObjectException {
+ public Database getDatabase(String catalogName, String name) throws NoSuchObjectException {
MetaException ex = null;
Database db = null;
try {
- db = getDatabaseInternal(name);
+ db = getDatabaseInternal(catalogName, name);
} catch (MetaException e) {
// Signature restriction to NSOE, and NSOE being a flat exception prevents us from
// setting the cause of the NSOE as the MetaException. We should not lose the info
@@ -850,32 +977,34 @@ public class ObjectStore implements RawStore, Configurable {
ex = e;
}
if (db == null) {
- LOG.warn("Failed to get database {}, returning NoSuchObjectException", name, ex);
+ LOG.warn("Failed to get database {}.{}, returning NoSuchObjectException",
+ catalogName, name, ex);
throw new NoSuchObjectException(name + (ex == null ? "" : (": " + ex.getMessage())));
}
return db;
}
- public Database getDatabaseInternal(String name) throws MetaException, NoSuchObjectException {
- return new GetDbHelper(name, true, true) {
+ public Database getDatabaseInternal(String catalogName, String name)
+ throws MetaException, NoSuchObjectException {
+ return new GetDbHelper(catalogName, name, true, true) {
@Override
protected Database getSqlResult(GetHelper<Database> ctx) throws MetaException {
- return directSql.getDatabase(dbName);
+ return directSql.getDatabase(catalogName, dbName);
}
@Override
protected Database getJdoResult(GetHelper<Database> ctx) throws MetaException, NoSuchObjectException {
- return getJDODatabase(dbName);
+ return getJDODatabase(catalogName, dbName);
}
}.run(false);
}
- public Database getJDODatabase(String name) throws NoSuchObjectException {
+ public Database getJDODatabase(String catName, String name) throws NoSuchObjectException {
MDatabase mdb = null;
boolean commited = false;
try {
openTransaction();
- mdb = getMDatabase(name);
+ mdb = getMDatabase(catName, name);
commited = commitTransaction();
} finally {
if (!commited) {
@@ -891,6 +1020,7 @@ public class ObjectStore implements RawStore, Configurable {
String type = org.apache.commons.lang.StringUtils.defaultIfBlank(mdb.getOwnerType(), null);
PrincipalType principalType = (type == null) ? null : PrincipalType.valueOf(type);
db.setOwnerType(principalType);
+ db.setCatalogName(catName);
return db;
}
@@ -903,13 +1033,13 @@ public class ObjectStore implements RawStore, Configurable {
* @throws NoSuchObjectException
*/
@Override
- public boolean alterDatabase(String dbName, Database db)
+ public boolean alterDatabase(String catName, String dbName, Database db)
throws MetaException, NoSuchObjectException {
MDatabase mdb = null;
boolean committed = false;
try {
- mdb = getMDatabase(dbName);
+ mdb = getMDatabase(catName, dbName);
mdb.setParameters(db.getParameters());
mdb.setOwnerName(db.getOwnerName());
if (db.getOwnerType() != null) {
@@ -934,19 +1064,21 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaException {
+ public boolean dropDatabase(String catName, String dbname)
+ throws NoSuchObjectException, MetaException {
boolean success = false;
- LOG.info("Dropping database {} along with all tables", dbname);
+ LOG.info("Dropping database {}.{} along with all tables", catName, dbname);
dbname = normalizeIdentifier(dbname);
+ catName = normalizeIdentifier(catName);
QueryWrapper queryWrapper = new QueryWrapper();
try {
openTransaction();
// then drop the database
- MDatabase db = getMDatabase(dbname);
+ MDatabase db = getMDatabase(catName, dbname);
pm.retrieve(db);
if (db != null) {
- List<MDBPrivilege> dbGrants = this.listDatabaseGrants(dbname, queryWrapper);
+ List<MDBPrivilege> dbGrants = this.listDatabaseGrants(catName, dbname, queryWrapper);
if (CollectionUtils.isNotEmpty(dbGrants)) {
pm.deletePersistentAll(dbGrants);
}
@@ -960,9 +1092,9 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<String> getDatabases(String pattern) throws MetaException {
+ public List<String> getDatabases(String catName, String pattern) throws MetaException {
if (pattern == null || pattern.equals("*")) {
- return getAllDatabases();
+ return getAllDatabases(catName);
}
boolean commited = false;
List<String> databases = null;
@@ -974,6 +1106,7 @@ public class ObjectStore implements RawStore, Configurable {
String[] subpatterns = pattern.trim().split("\\|");
StringBuilder filterBuilder = new StringBuilder();
List<String> parameterVals = new ArrayList<>(subpatterns.length);
+ appendSimpleCondition(filterBuilder, "catalogName", new String[] {catName}, parameterVals);
appendPatternCondition(filterBuilder, "name", subpatterns, parameterVals);
query = pm.newQuery(MDatabase.class, filterBuilder.toString());
query.setResult("name");
@@ -988,18 +1121,20 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<String> getAllDatabases() throws MetaException {
+ public List<String> getAllDatabases(String catName) throws MetaException {
boolean commited = false;
List<String> databases = null;
- String queryStr = "select name from org.apache.hadoop.hive.metastore.model.MDatabase";
Query query = null;
+ catName = normalizeIdentifier(catName);
openTransaction();
try {
- query = pm.newQuery(queryStr);
+ query = pm.newQuery("select name from org.apache.hadoop.hive.metastore.model.MDatabase " +
+ "where catalogName == catname");
+ query.declareParameters("java.lang.String catname");
query.setResult("name");
- databases = new ArrayList<>((Collection<String>) query.execute());
+ databases = new ArrayList<>((Collection<String>) query.execute(catName));
commited = commitTransaction();
} finally {
rollbackAndCleanup(commited, query);
@@ -1112,12 +1247,25 @@ public class ObjectStore implements RawStore, Configurable {
// Add constraints.
// We need not do a deep retrieval of the Table Column Descriptor while persisting the
// constraints since this transaction involving create table is not yet committed.
- List<String> constraintNames = addForeignKeys(foreignKeys, false, primaryKeys, uniqueConstraints);
- constraintNames.addAll(addPrimaryKeys(primaryKeys, false));
- constraintNames.addAll(addUniqueConstraints(uniqueConstraints, false));
- constraintNames.addAll(addNotNullConstraints(notNullConstraints, false));
- constraintNames.addAll(addDefaultConstraints(defaultConstraints, false));
- constraintNames.addAll(addCheckConstraints(checkConstraints, false));
+ List<String> constraintNames = new ArrayList<>();
+ if (foreignKeys != null) {
+ constraintNames.addAll(addForeignKeys(foreignKeys, false, primaryKeys, uniqueConstraints));
+ }
+ if (primaryKeys != null) {
+ constraintNames.addAll(addPrimaryKeys(primaryKeys, false));
+ }
+ if (uniqueConstraints != null) {
+ constraintNames.addAll(addUniqueConstraints(uniqueConstraints, false));
+ }
+ if (notNullConstraints != null) {
+ constraintNames.addAll(addNotNullConstraints(notNullConstraints, false));
+ }
+ if (defaultConstraints != null) {
+ constraintNames.addAll(addDefaultConstraints(defaultConstraints, false));
+ }
+ if (checkConstraints != null) {
+ constraintNames.addAll(addCheckConstraints(checkConstraints, false));
+ }
success = commitTransaction();
return constraintNames;
} finally {
@@ -1205,47 +1353,47 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public boolean dropTable(String dbName, String tableName) throws MetaException,
- NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ public boolean dropTable(String catName, String dbName, String tableName)
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
boolean materializedView = false;
boolean success = false;
try {
openTransaction();
- MTable tbl = getMTable(dbName, tableName);
+ MTable tbl = getMTable(catName, dbName, tableName);
pm.retrieve(tbl);
if (tbl != null) {
materializedView = TableType.MATERIALIZED_VIEW.toString().equals(tbl.getTableType());
// first remove all the grants
- List<MTablePrivilege> tabGrants = listAllTableGrants(dbName, tableName);
+ List<MTablePrivilege> tabGrants = listAllTableGrants(catName, dbName, tableName);
if (CollectionUtils.isNotEmpty(tabGrants)) {
pm.deletePersistentAll(tabGrants);
}
- List<MTableColumnPrivilege> tblColGrants = listTableAllColumnGrants(dbName,
+ List<MTableColumnPrivilege> tblColGrants = listTableAllColumnGrants(catName, dbName,
tableName);
if (CollectionUtils.isNotEmpty(tblColGrants)) {
pm.deletePersistentAll(tblColGrants);
}
- List<MPartitionPrivilege> partGrants = this.listTableAllPartitionGrants(dbName, tableName);
+ List<MPartitionPrivilege> partGrants = this.listTableAllPartitionGrants(catName, dbName, tableName);
if (CollectionUtils.isNotEmpty(partGrants)) {
pm.deletePersistentAll(partGrants);
}
- List<MPartitionColumnPrivilege> partColGrants = listTableAllPartitionColumnGrants(dbName,
+ List<MPartitionColumnPrivilege> partColGrants = listTableAllPartitionColumnGrants(catName, dbName,
tableName);
if (CollectionUtils.isNotEmpty(partColGrants)) {
pm.deletePersistentAll(partColGrants);
}
// delete column statistics if present
try {
- deleteTableColumnStatistics(dbName, tableName, null);
+ deleteTableColumnStatistics(catName, dbName, tableName, null);
} catch (NoSuchObjectException e) {
- LOG.info("Found no table level column statistics associated with db {}" +
- " table {} record to delete", dbName, tableName);
+ LOG.info("Found no table level column statistics associated with {} to delete",
+ getCatalogQualifiedTableName(catName, dbName, tableName));
}
List<MConstraint> tabConstraints = listAllTableConstraintsWithOptionalConstraintName(
- dbName, tableName, null);
+ catName, dbName, tableName, null);
if (CollectionUtils.isNotEmpty(tabConstraints)) {
pm.deletePersistentAll(tabConstraints);
}
@@ -1253,7 +1401,7 @@ public class ObjectStore implements RawStore, Configurable {
preDropStorageDescriptor(tbl.getSd());
if (materializedView) {
- dropCreationMetadata(
+ dropCreationMetadata(tbl.getDatabase().getCatalogName(),
tbl.getDatabase().getName(), tbl.getTableName());
}
@@ -1273,14 +1421,14 @@ public class ObjectStore implements RawStore, Configurable {
return success;
}
- private boolean dropCreationMetadata(String dbName, String tableName) throws MetaException,
+ private boolean dropCreationMetadata(String catName, String dbName, String tableName) throws MetaException,
NoSuchObjectException, InvalidObjectException, InvalidInputException {
boolean success = false;
dbName = normalizeIdentifier(dbName);
tableName = normalizeIdentifier(tableName);
try {
openTransaction();
- MCreationMetadata mcm = getCreationMetadata(dbName, tableName);
+ MCreationMetadata mcm = getCreationMetadata(catName, dbName, tableName);
pm.retrieve(mcm);
if (mcm != null) {
pm.deletePersistentAll(mcm);
@@ -1294,8 +1442,9 @@ public class ObjectStore implements RawStore, Configurable {
return success;
}
- private List<MConstraint> listAllTableConstraintsWithOptionalConstraintName
- (String dbName, String tableName, String constraintname) {
+ private List<MConstraint> listAllTableConstraintsWithOptionalConstraintName(
+ String catName, String dbName, String tableName, String constraintname) {
+ catName = normalizeIdentifier(catName);
dbName = normalizeIdentifier(dbName);
tableName = normalizeIdentifier(tableName);
constraintname = constraintname!=null?normalizeIdentifier(constraintname):null;
@@ -1305,19 +1454,21 @@ public class ObjectStore implements RawStore, Configurable {
try {
query = pm.newQuery("select constraintName from org.apache.hadoop.hive.metastore.model.MConstraint where "
- + "((parentTable.tableName == ptblname && parentTable.database.name == pdbname) || "
- + "(childTable != null && childTable.tableName == ctblname && "
- + "childTable.database.name == cdbname)) " + (constraintname != null ?
- " && constraintName == constraintname" : ""));
+ + "((parentTable.tableName == ptblname && parentTable.database.name == pdbname && " +
+ "parentTable.database.catalogName == pcatname) || "
+ + "(childTable != null && childTable.tableName == ctblname &&" +
+ "childTable.database.name == cdbname && childTable.database.catalogName == ccatname)) " +
+ (constraintname != null ? " && constraintName == constraintname" : ""));
query.declareParameters("java.lang.String ptblname, java.lang.String pdbname,"
- + "java.lang.String ctblname, java.lang.String cdbname" +
+ + "java.lang.String pcatname, java.lang.String ctblname, java.lang.String cdbname," +
+ "java.lang.String ccatname" +
(constraintname != null ? ", java.lang.String constraintname" : ""));
Collection<?> constraintNamesColl =
constraintname != null ?
((Collection<?>) query.
- executeWithArray(tableName, dbName, tableName, dbName, constraintname)):
+ executeWithArray(tableName, dbName, catName, tableName, dbName, catName, constraintname)):
((Collection<?>) query.
- executeWithArray(tableName, dbName, tableName, dbName));
+ executeWithArray(tableName, dbName, catName, tableName, dbName, catName));
for (Iterator<?> i = constraintNamesColl.iterator(); i.hasNext();) {
String currName = (String) i.next();
constraintNames.add(currName);
@@ -1340,16 +1491,16 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public Table getTable(String dbName, String tableName) throws MetaException {
+ public Table getTable(String catName, String dbName, String tableName) throws MetaException {
boolean commited = false;
Table tbl = null;
try {
openTransaction();
- tbl = convertToTable(getMTable(dbName, tableName));
+ tbl = convertToTable(getMTable(catName, dbName, tableName));
// Retrieve creation metadata if needed
if (tbl != null && TableType.MATERIALIZED_VIEW.toString().equals(tbl.getTableType())) {
tbl.setCreationMetadata(
- convertToCreationMetadata(getCreationMetadata(dbName, tableName)));
+ convertToCreationMetadata(getCreationMetadata(catName, dbName, tableName)));
}
commited = commitTransaction();
} finally {
@@ -1361,40 +1512,46 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<String> getTables(String dbName, String pattern) throws MetaException {
- return getTables(dbName, pattern, null);
+ public List<String> getTables(String catName, String dbName, String pattern)
+ throws MetaException {
+ return getTables(catName, dbName, pattern, null);
}
@Override
- public List<String> getTables(String dbName, String pattern, TableType tableType) throws MetaException {
+ public List<String> getTables(String catName, String dbName, String pattern, TableType tableType)
+ throws MetaException {
try {
// We only support pattern matching via jdo since pattern matching in Java
// might be different than the one used by the metastore backends
- return getTablesInternal(dbName, pattern, tableType, (pattern == null || pattern.equals(".*")), true);
+ return getTablesInternal(catName, dbName, pattern, tableType,
+ (pattern == null || pattern.equals(".*")), true);
} catch (NoSuchObjectException e) {
throw new MetaException(ExceptionUtils.getStackTrace(e));
}
}
- protected List<String> getTablesInternal(String dbName, String pattern, TableType tableType,
- boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException {
+ protected List<String> getTablesInternal(String catName, String dbName, String pattern,
+ TableType tableType, boolean allowSql, boolean allowJdo)
+ throws MetaException, NoSuchObjectException {
final String db_name = normalizeIdentifier(dbName);
- return new GetListHelper<String>(dbName, null, allowSql, allowJdo) {
+ final String cat_name = normalizeIdentifier(catName);
+ return new GetListHelper<String>(cat_name, dbName, null, allowSql, allowJdo) {
@Override
protected List<String> getSqlResult(GetHelper<List<String>> ctx)
throws MetaException {
- return directSql.getTables(db_name, tableType);
+ return directSql.getTables(cat_name, db_name, tableType);
}
@Override
protected List<String> getJdoResult(GetHelper<List<String>> ctx)
throws MetaException, NoSuchObjectException {
- return getTablesInternalViaJdo(db_name, pattern, tableType);
+ return getTablesInternalViaJdo(cat_name, db_name, pattern, tableType);
}
}.run(false);
}
- private List<String> getTablesInternalViaJdo(String dbName, String pattern, TableType tableType) throws MetaException {
+ private List<String> getTablesInternalViaJdo(String catName, String dbName, String pattern,
+ TableType tableType) throws MetaException {
boolean commited = false;
Query query = null;
List<String> tbls = null;
@@ -1407,6 +1564,7 @@ public class ObjectStore implements RawStore, Configurable {
StringBuilder filterBuilder = new StringBuilder();
//adds database.name == dbName to the filter
appendSimpleCondition(filterBuilder, "database.name", new String[] {dbName}, parameterVals);
+ appendSimpleCondition(filterBuilder, "database.catalogName", new String[] {catName}, parameterVals);
if(pattern != null) {
appendPatternCondition(filterBuilder, "tableName", pattern, parameterVals);
}
@@ -1427,21 +1585,23 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<String> getMaterializedViewsForRewriting(String dbName)
+ public List<String> getMaterializedViewsForRewriting(String catName, String dbName)
throws MetaException, NoSuchObjectException {
final String db_name = normalizeIdentifier(dbName);
+ catName = normalizeIdentifier(catName);
boolean commited = false;
Query<?> query = null;
List<String> tbls = null;
try {
openTransaction();
dbName = normalizeIdentifier(dbName);
- query = pm.newQuery(MTable.class, "database.name == db && tableType == tt"
- + " && rewriteEnabled == re");
- query.declareParameters("java.lang.String db, java.lang.String tt, boolean re");
+ query = pm.newQuery(MTable.class,
+ "database.name == db && database.catalogName == cat && tableType == tt && rewriteEnabled == re");
+ query.declareParameters(
+ "java.lang.String db, java.lang.String cat, java.lang.String tt, boolean re");
query.setResult("tableName");
- Collection<String> names = (Collection<String>) query.execute(
- db_name, TableType.MATERIALIZED_VIEW.toString(), true);
+ Collection<String> names = (Collection<String>) query.executeWithArray(
+ db_name, catName, TableType.MATERIALIZED_VIEW.toString(), true);
tbls = new ArrayList<>(names);
commited = commitTransaction();
} finally {
@@ -1483,8 +1643,8 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes)
- throws MetaException {
+ public List<TableMeta> getTableMeta(String catName, String dbNames, String tableNames,
+ List<String> tableTypes) throws MetaException {
boolean commited = false;
Query query = null;
@@ -1495,6 +1655,7 @@ public class ObjectStore implements RawStore, Configurable {
// patterns
StringBuilder filterBuilder = new StringBuilder();
List<String> parameterVals = new ArrayList<>();
+ appendSimpleCondition(filterBuilder, "database.catalogName", new String[] {catName}, parameterVals);
if (dbNames != null && !dbNames.equals("*")) {
appendPatternCondition(filterBuilder, "database.name", dbNames, parameterVals);
}
@@ -1505,6 +1666,10 @@ public class ObjectStore implements RawStore, Configurable {
appendSimpleCondition(filterBuilder, "tableType", tableTypes.toArray(new String[0]), parameterVals);
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getTableMeta with filter " + filterBuilder.toString() + " params: " +
+ StringUtils.join(parameterVals, ", "));
+ }
query = pm.newQuery(MTable.class, filterBuilder.toString());
Collection<MTable> tables = (Collection<MTable>) query.executeWithArray(parameterVals.toArray(new String[parameterVals.size()]));
for (MTable table : tables) {
@@ -1563,8 +1728,8 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<String> getAllTables(String dbName) throws MetaException {
- return getTables(dbName, ".*");
+ public List<String> getAllTables(String catName, String dbName) throws MetaException {
+ return getTables(catName, dbName, ".*");
}
class AttachedMTableInfo {
@@ -1579,19 +1744,25 @@ public class ObjectStore implements RawStore, Configurable {
}
}
- private AttachedMTableInfo getMTable(String db, String table, boolean retrieveCD) {
+ private AttachedMTableInfo getMTable(String catName, String db, String table,
+ boolean retrieveCD) {
AttachedMTableInfo nmtbl = new AttachedMTableInfo();
MTable mtbl = null;
boolean commited = false;
Query query = null;
try {
openTransaction();
+ catName = normalizeIdentifier(catName);
db = normalizeIdentifier(db);
table = normalizeIdentifier(table);
- query = pm.newQuery(MTable.class, "tableName == table && database.name == db");
- query.declareParameters("java.lang.String table, java.lang.String db");
+ query = pm.newQuery(MTable.class,
+ "tableName == table && database.name == db && database.catalogName == catname");
+ query.declareParameters(
+ "java.lang.String table, java.lang.String db, java.lang.String catname");
query.setUnique(true);
- mtbl = (MTable) query.execute(table, db);
+ LOG.debug("Executing getMTable for " +
+ getCatalogQualifiedTableName(catName, db, table));
+ mtbl = (MTable) query.execute(table, db, catName);
pm.retrieve(mtbl);
// Retrieving CD can be expensive and unnecessary, so do it only when required.
if (mtbl != null && retrieveCD) {
@@ -1607,17 +1778,17 @@ public class ObjectStore implements RawStore, Configurable {
return nmtbl;
}
- private MCreationMetadata getCreationMetadata(String dbName, String tblName) {
+ private MCreationMetadata getCreationMetadata(String catName, String dbName, String tblName) {
boolean commited = false;
MCreationMetadata mcm = null;
Query query = null;
try {
openTransaction();
query = pm.newQuery(
- MCreationMetadata.class, "tblName == table && dbName == db");
- query.declareParameters("java.lang.String table, java.lang.String db");
+ MCreationMetadata.class, "tblName == table && dbName == db && catalogName == cat");
+ query.declareParameters("java.lang.String table, java.lang.String db, java.lang.String cat");
query.setUnique(true);
- mcm = (MCreationMetadata) query.execute(tblName, dbName);
+ mcm = (MCreationMetadata) query.execute(tblName, dbName, catName);
pm.retrieve(mcm);
commited = commitTransaction();
} finally {
@@ -1626,14 +1797,14 @@ public class ObjectStore implements RawStore, Configurable {
return mcm;
}
- private MTable getMTable(String db, String table) {
- AttachedMTableInfo nmtbl = getMTable(db, table, false);
+ private MTable getMTable(String catName, String db, String table) {
+ AttachedMTableInfo nmtbl = getMTable(catName, db, table, false);
return nmtbl.mtbl;
}
@Override
- public List<Table> getTableObjectsByName(String db, List<String> tbl_names) throws MetaException,
- UnknownDBException {
+ public List<Table> getTableObjectsByName(String catName, String db, List<String> tbl_names)
+ throws MetaException, UnknownDBException {
List<Table> tables = new ArrayList<>();
boolean committed = false;
Query dbExistsQuery = null;
@@ -1641,25 +1812,31 @@ public class ObjectStore implements RawStore, Configurable {
try {
openTransaction();
db = normalizeIdentifier(db);
- dbExistsQuery = pm.newQuery(MDatabase.class, "name == db");
- dbExistsQuery.declareParameters("java.lang.String db");
- dbExistsQuery.setUnique(true);
- dbExistsQuery.setResult("name");
- String dbNameIfExists = (String) dbExistsQuery.execute(db);
- if (org.apache.commons.lang.StringUtils.isEmpty(dbNameIfExists)) {
- throw new UnknownDBException("Could not find database " + db);
- }
+ catName = normalizeIdentifier(catName);
List<String> lowered_tbl_names = new ArrayList<>(tbl_names.size());
for (String t : tbl_names) {
lowered_tbl_names.add(normalizeIdentifier(t));
}
query = pm.newQuery(MTable.class);
- query.setFilter("database.name == db && tbl_names.contains(tableName)");
- query.declareParameters("java.lang.String db, java.util.Collection tbl_names");
- Collection mtables = (Collection) query.execute(db, lowered_tbl_names);
- for (Iterator iter = mtables.iterator(); iter.hasNext();) {
- tables.add(convertToTable((MTable) iter.next()));
+ query.setFilter("database.name == db && database.catalogName == cat && tbl_names.contains(tableName)");
+ query.declareParameters("java.lang.String db, java.lang.String cat, java.util.Collection tbl_names");
+ Collection mtables = (Collection) query.execute(db, catName, lowered_tbl_names);
+ if (mtables == null || mtables.isEmpty()) {
+ // Need to differentiate between an unmatched pattern and a non-existent database
+ dbExistsQuery = pm.newQuery(MDatabase.class, "name == db && catalogName == cat");
+ dbExistsQuery.declareParameters("java.lang.String db, java.lang.String cat");
+ dbExistsQuery.setUnique(true);
+ dbExistsQuery.setResult("name");
+ String dbNameIfExists = (String) dbExistsQuery.execute(db, catName);
+ if (org.apache.commons.lang.StringUtils.isEmpty(dbNameIfExists)) {
+ throw new UnknownDBException("Could not find database " +
+ getCatalogQualifiedDbName(catName, db));
+ }
+ } else {
+ for (Iterator iter = mtables.iterator(); iter.hasNext(); ) {
+ tables.add(convertToTable((MTable) iter.next()));
+ }
}
committed = commitTransaction();
} finally {
@@ -1703,6 +1880,7 @@ public class ObjectStore implements RawStore, Configurable {
convertToFieldSchemas(mtbl.getPartitionKeys()), convertMap(mtbl.getParameters()),
mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType);
t.setRewriteEnabled(mtbl.isRewriteEnabled());
+ t.setCatName(mtbl.getDatabase().getCatalogName());
return t;
}
@@ -1712,12 +1890,13 @@ public class ObjectStore implements RawStore, Configurable {
return null;
}
MDatabase mdb = null;
+ String catName = tbl.isSetCatName() ? tbl.getCatName() : getDefaultCatalog(conf);
try {
- mdb = getMDatabase(tbl.getDbName());
+ mdb = getMDatabase(catName, tbl.getDbName());
} catch (NoSuchObjectException e) {
LOG.error("Could not convert to MTable", e);
- throw new InvalidObjectException("Database " + tbl.getDbName()
- + " doesn't exist.");
+ throw new InvalidObjectException("Database " +
+ getCatalogQualifiedDbName(catName, tbl.getDbName()) + " doesn't exist.");
}
// If the table has property EXTERNAL set, update table type
@@ -1965,9 +2144,9 @@ public class ObjectStore implements RawStore, Configurable {
Set<MTable> tablesUsed = new HashSet<>();
for (String fullyQualifiedName : m.getTablesUsed()) {
String[] names = fullyQualifiedName.split("\\.");
- tablesUsed.add(getMTable(names[0], names[1], false).mtbl);
+ tablesUsed.add(getMTable(m.getCatName(), names[0], names[1], false).mtbl);
}
- return new MCreationMetadata(m.getDbName(), m.getTblName(),
+ return new MCreationMetadata(m.getCatName(), m.getDbName(), m.getTblName(),
tablesUsed, m.getValidTxnList());
}
@@ -1982,7 +2161,7 @@ public class ObjectStore implements RawStore, Configurable {
Warehouse.getQualifiedName(
mtbl.getDatabase().getName(), mtbl.getTableName()));
}
- CreationMetadata r = new CreationMetadata(
+ CreationMetadata r = new CreationMetadata(s.getCatalogName(),
s.getDbName(), s.getTblName(), tablesUsed);
if (s.getTxnList() != null) {
r.setValidTxnList(s.getTxnList());
@@ -1991,17 +2170,17 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public boolean addPartitions(String dbName, String tblName, List<Partition> parts)
+ public boolean addPartitions(String catName, String dbName, String tblName, List<Partition> parts)
throws InvalidObjectException, MetaException {
boolean success = false;
openTransaction();
try {
List<MTablePrivilege> tabGrants = null;
List<MTableColumnPrivilege> tabColumnGrants = null;
- MTable table = this.getMTable(dbName, tblName);
+ MTable table = this.getMTable(catName, dbName, tblName);
if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
- tabGrants = this.listAllTableGrants(dbName, tblName);
- tabColumnGrants = this.listTableAllColumnGrants(dbName, tblName);
+ tabGrants = this.listAllTableGrants(catName, dbName, tblName);
+ tabColumnGrants = this.listTableAllColumnGrants(catName, dbName, tblName);
}
List<Object> toPersist = new ArrayList<>();
for (Partition part : parts) {
@@ -2046,7 +2225,7 @@ public class ObjectStore implements RawStore, Configurable {
Partition part, boolean ifNotExists) throws MetaException {
MetaStoreUtils.validatePartitionNameCharacters(part.getValues(),
partitionValidationPattern);
- boolean doesExist = doesPartitionExist(
+ boolean doesExist = doesPartitionExist(part.getCatName(),
part.getDbName(), part.getTableName(), part.getValues());
if (doesExist && !ifNotExists) {
throw new MetaException("Partition already exists: " + part);
@@ -2055,7 +2234,7 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public boolean addPartitions(String dbName, String tblName,
+ public boolean addPartitions(String catName, String dbName, String tblName,
PartitionSpecProxy partitionSpec, boolean ifNotExists)
throws InvalidObjectException, MetaException {
boolean success = false;
@@ -2063,10 +2242,10 @@ public class ObjectStore implements RawStore, Configurable {
try {
List<MTablePrivilege> tabGrants = null;
List<MTableColumnPrivilege> tabColumnGrants = null;
- MTable table = this.getMTable(dbName, tblName);
+ MTable table = this.getMTable(catName, dbName, tblName);
if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
- tabGrants = this.listAllTableGrants(dbName, tblName);
- tabColumnGrants = this.listTableAllColumnGrants(dbName, tblName);
+ tabGrants = this.listAllTableGrants(catName, dbName, tblName);
+ tabColumnGrants = this.listTableAllColumnGrants(catName, dbName, tblName);
}
if (!partitionSpec.getTableName().equals(tblName) || !partitionSpec.getDbName().equals(dbName)) {
@@ -2117,14 +2296,14 @@ public class ObjectStore implements RawStore, Configurable {
boolean success = false;
boolean commited = false;
try {
- MTable table = this.getMTable(part.getDbName(), part.getTableName());
+ String catName = part.isSetCatName() ? part.getCatName() : getDefaultCatalog(conf);
+ MTable table = this.getMTable(catName, part.getDbName(), part.getTableName());
List<MTablePrivilege> tabGrants = null;
List<MTableColumnPrivilege> tabColumnGrants = null;
if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
- tabGrants = this.listAllTableGrants(part
- .getDbName(), part.getTableName());
+ tabGrants = this.listAllTableGrants(catName, part.getDbName(), part.getTableName());
tabColumnGrants = this.listTableAllColumnGrants(
- part.getDbName(), part.getTableName());
+ catName, part.getDbName(), part.getTableName());
}
openTransaction();
MPartition mpart = convertToMPart(part, true);
@@ -2167,10 +2346,10 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public Partition getPartition(String dbName, String tableName,
+ public Partition getPartition(String catName, String dbName, String tableName,
List<String> part_vals) throws NoSuchObjectException, MetaException {
openTransaction();
- Partition part = convertToPart(getMPartition(dbName, tableName, part_vals));
+ Partition part = convertToPart(getMPartition(catName, dbName, tableName, part_vals));
commitTransaction();
if(part == null) {
throw new NoSuchObjectException("partition values="
@@ -2180,7 +2359,7 @@ public class ObjectStore implements RawStore, Configurable {
return part;
}
- private MPartition getMPartition(String dbName, String tableName, List<String> part_vals)
+ private MPartition getMPartition(String catName, String dbName, String tableName, List<String> part_vals)
throws MetaException {
List<MPartition> mparts = null;
MPartition ret = null;
@@ -2188,9 +2367,10 @@ public class ObjectStore implements RawStore, Configurable {
Query query = null;
try {
openTransaction();
+ catName = normalizeIdentifier(catName);
dbName = normalizeIdentifier(dbName);
tableName = normalizeIdentifier(tableName);
- MTable mtbl = getMTable(dbName, tableName);
+ MTable mtbl = getMTable(catName, dbName, tableName);
if (mtbl == null) {
commited = commitTransaction();
return null;
@@ -2201,9 +2381,11 @@ public class ObjectStore implements RawStore, Configurable {
Warehouse.makePartName(convertToFieldSchemas(mtbl.getPartitionKeys()), part_vals);
query =
pm.newQuery(MPartition.class,
- "table.tableName == t1 && table.database.name == t2 && partitionName == t3");
- query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
- mparts = (List<MPartition>) query.execute(tableName, dbName, name);
+ "table.tableName == t1 && table.database.name == t2 && partitionName == t3 " +
+ " && table.database.catalogName == t4");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, "
+ + "java.lang.String t4");
+ mparts = (List<MPartition>) query.executeWithArray(tableName, dbName, name, catName);
pm.retrieveAll(mparts);
commited = commitTransaction();
// We need to compare partition name with requested name since some DBs
@@ -2246,7 +2428,7 @@ public class ObjectStore implements RawStore, Configurable {
if (part == null) {
return null;
}
- MTable mt = getMTable(part.getDbName(), part.getTableName());
+ MTable mt = getMTable(part.getCatName(), part.getDbName(), part.getTableName());
if (mt == null) {
throw new InvalidObjectException(
"Partition doesn't have a valid table or database name");
@@ -2277,30 +2459,34 @@ public class ObjectStore implements RawStore, Configurable {
if (mpart == null) {
return null;
}
- return new Partition(convertList(mpart.getValues()), mpart.getTable().getDatabase()
+ Partition p = new Partition(convertList(mpart.getValues()), mpart.getTable().getDatabase()
.getName(), mpart.getTable().getTableName(), mpart.getCreateTime(),
mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd()),
convertMap(mpart.getParameters()));
+ p.setCatName(mpart.getTable().getDatabase().getCatalogName());
+ return p;
}
- private Partition convertToPart(String dbName, String tblName, MPartition mpart)
+ private Partition convertToPart(String catName, String dbName, String tblName, MPartition mpart)
throws MetaException {
if (mpart == null) {
return null;
}
- return new Partition(convertList(mpart.getValues()), dbName, tblName,
+ Partition p = new Partition(convertList(mpart.getValues()), dbName, tblName,
mpart.getCreateTime(), mpart.getLastAccessTime(),
convertToStorageDescriptor(mpart.getSd(), false), convertMap(mpart.getParameters()));
+ p.setCatName(catName);
+ return p;
}
@Override
- public boolean dropPartition(String dbName, String tableName,
+ public boolean dropPartition(String catName, String dbName, String tableName,
List<String> part_vals) throws MetaException, NoSuchObjectException, InvalidObjectException,
InvalidInputException {
boolean success = false;
try {
openTransaction();
- MPartition part = getMPartition(dbName, tableName, part_vals);
+ MPartition part = getMPartition(catName, dbName, tableName, part_vals);
dropPartitionCommon(part);
success = commitTransaction();
} finally {
@@ -2312,7 +2498,7 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public void dropPartitions(String dbName, String tblName, List<String> partNames)
+ public void dropPartitions(String catName, String dbName, String tblName, List<String> partNames)
throws MetaException, NoSuchObjectException {
if (CollectionUtils.isEmpty(partNames)) {
return;
@@ -2321,15 +2507,15 @@ public class ObjectStore implements RawStore, Configurable {
openTransaction();
try {
// Delete all things.
- dropPartitionGrantsNoTxn(dbName, tblName, partNames);
- dropPartitionAllColumnGrantsNoTxn(dbName, tblName, partNames);
- dropPartitionColumnStatisticsNoTxn(dbName, tblName, partNames);
+ dropPartitionGrantsNoTxn(catName, dbName, tblName, partNames);
+ dropPartitionAllColumnGrantsNoTxn(catName, dbName, tblName, partNames);
+ dropPartitionColumnStatisticsNoTxn(catName, dbName, tblName, partNames);
// CDs are reused; go thry partition SDs, detach all CDs from SDs, then remove unused CDs.
- for (MColumnDescriptor mcd : detachCdsFromSdsNoTxn(dbName, tblName, partNames)) {
+ for (MColumnDescriptor mcd : detachCdsFromSdsNoTxn(catName, dbName, tblName, partNames)) {
removeUnusedColumnDescriptor(mcd);
}
- dropPartitionsNoTxn(dbName, tblName, partNames);
+ dropPartitionsNoTxn(catName, dbName, tblName, partNames);
if (!(success = commitTransaction())) {
throw new MetaException("Failed to drop partitions"); // Should not happen?
}
@@ -2343,12 +2529,6 @@ public class ObjectStore implements RawStore, Configurable {
/**
* Drop an MPartition and cascade deletes (e.g., delete partition privilege grants,
* drop the storage descriptor cleanly, etc.)
- * @param part - the MPartition to drop
- * @return whether the transaction committed successfully
- * @throws InvalidInputException
- * @throws InvalidObjectException
- * @throws MetaException
- * @throws NoSuchObjectException
*/
private boolean dropPartitionCommon(MPartition part) throws NoSuchObjectException, MetaException,
InvalidObjectException, InvalidInputException {
@@ -2364,6 +2544,7 @@ public class ObjectStore implements RawStore, Configurable {
String partName = FileUtils.makePartName(colNames, part.getValues());
List<MPartitionPrivilege> partGrants = listPartitionGrants(
+ part.getTable().getDatabase().getCatalogName(),
part.getTable().getDatabase().getName(),
part.getTable().getTableName(),
Lists.newArrayList(partName));
@@ -2373,6 +2554,7 @@ public class ObjectStore implements RawStore, Configurable {
}
List<MPartitionColumnPrivilege> partColumnGrants = listPartitionAllColumnGrants(
+ part.getTable().getDatabase().getCatalogName(),
part.getTable().getDatabase().getName(),
part.getTable().getTableName(),
Lists.newArrayList(partName));
@@ -2380,12 +2562,13 @@ public class ObjectStore implements RawStore, Configurable {
pm.deletePersistentAll(partColumnGrants);
}
+ String catName = part.getTable().getDatabase().getCatalogName();
String dbName = part.getTable().getDatabase().getName();
String tableName = part.getTable().getTableName();
// delete partition level column stats if it exists
try {
- deletePartitionColumnStatistics(dbName, tableName, partName, part.getValues(), null);
+ deletePartitionColumnStatistics(catName, dbName, tableName, partName, part.getValues(), null);
} catch (NoSuchObjectException e) {
LOG.info("No column statistics records found to delete");
}
@@ -2403,26 +2586,26 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<Partition> getPartitions(
- String dbName, String tableName, int maxParts) throws MetaException, NoSuchObjectException {
- return getPartitionsInternal(dbName, tableName, maxParts, true, true);
+ public List<Partition> getPartitions(String catName, String dbName, String tableName,
+ int maxParts) throws MetaException, NoSuchObjectException {
+ return getPartitionsInternal(catName, dbName, tableName, maxParts, true, true);
}
- protected List<Partition> getPartitionsInternal(
- String dbName, String tblName, final int maxParts, boolean allowSql, boolean allowJdo)
+ protected List<Partition> getPartitionsInternal(String catName, String dbName, String tblName,
+ final int maxParts, boolean allowSql, boolean allowJdo)
throws MetaException, NoSuchObjectException {
- return new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) {
+ return new GetListHelper<Partition>(catName, dbName, tblName, allowSql, allowJdo) {
@Override
protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException {
Integer max = (maxParts < 0) ? null : maxParts;
- return directSql.getPartitions(dbName, tblName, max);
+ return directSql.getPartitions(catName, dbName, tblName, max);
}
@Override
protected List<Partition> getJdoResult(
GetHelper<List<Partition>> ctx) throws MetaException {
QueryWrapper queryWrapper = new QueryWrapper();
try {
- return convertToParts(listMPartitions(dbName, tblName, maxParts, queryWrapper));
+ return convertToParts(listMPartitions(catName, dbName, tblName, maxParts, queryWrapper));
} finally {
queryWrapper.close();
}
@@ -2431,7 +2614,7 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<Partition> getPartitionsWithAuth(String dbName, String tblName,
+ public List<Partition> getPartitionsWithAuth(String catName, String dbName, String tblName,
short max, String userName, List<String> groupNames)
throws MetaException, InvalidObjectException {
boolean success = false;
@@ -2439,7 +2622,7 @@ public class ObjectStore implements RawStore, Configurable {
try {
openTransaction();
- List<MPartition> mparts = listMPartitions(dbName, tblName, max, queryWrapper);
+ List<MPartition> mparts = listMPartitions(catName, dbName, tblName, max, queryWrapper);
List<Partition> parts = new ArrayList<>(mparts.size());
if (CollectionUtils.isNotEmpty(mparts)) {
for (MPartition mpart : mparts) {
@@ -2450,7 +2633,7 @@ public class ObjectStore implements RawStore, Configurable {
if ("TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
String partName = Warehouse.makePartName(this.convertToFieldSchemas(mtbl
.getPartitionKeys()), part.getValues());
- PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(dbName,
+ PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(catName, dbName,
tblName, partName, userName, groupNames);
part.setPrivileges(partAuth);
}
@@ -2464,13 +2647,13 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public Partition getPartitionWithAuth(String dbName, String tblName,
+ public Partition getPartitionWithAuth(String catName, String dbName, String tblName,
List<String> partVals, String user_name, List<String> group_names)
throws NoSuchObjectException, MetaException, InvalidObjectException {
boolean success = false;
try {
openTransaction();
- MPartition mpart = getMPartition(dbName, tblName, partVals);
+ MPartition mpart = getMPartition(catName, dbName, tblName, partVals);
if (mpart == null) {
commitTransaction();
throw new NoSuchObjectException("partition values="
@@ -2482,7 +2665,7 @@ public class ObjectStore implements RawStore, Configurable {
if ("TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
String partName = Warehouse.makePartName(this.convertToFieldSchemas(mtbl
.getPartitionKeys()), partVals);
- PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(dbName,
+ PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(catName, dbName,
tblName, partName, user_name, group_names);
part.setPrivileges(partAuth);
}
@@ -2515,11 +2698,11 @@ public class ObjectStore implements RawStore, Configurable {
return dest;
}
- private List<Partition> convertToParts(String dbName, String tblName, List<MPartition> mparts)
+ private List<Partition> convertToParts(String catName, String dbName, String tblName, List<MPartition> mparts)
throws MetaException {
List<Partition> parts = new ArrayList<>(mparts.size());
for (MPartition mp : mparts) {
- parts.add(convertToPart(dbName, tblName, mp));
+ parts.add(convertToPart(catName, dbName, tblName, mp));
Deadline.checkTimeout();
}
return parts;
@@ -2527,14 +2710,14 @@ public class ObjectStore implements RawStore, Configurable {
// TODO:pc implement max
@Override
- public List<String> listPartitionNames(String dbName, String tableName,
+ public List<String> listPartitionNames(String catName, String dbName, String tableName,
short max) throws MetaException {
List<String> pns = null;
boolean success = false;
try {
openTransaction();
LOG.debug("Executing getPartitionNames");
- pns = getPartitionNamesNoTxn(dbName, tableName, max);
+ pns = getPartitionNamesNoTxn(catName, dbName, tableName, max);
success = commitTransaction();
} finally {
if (!success) {
@@ -2584,21 +2767,24 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public PartitionValuesResponse listPartitionValues(String dbName, String tableName, List<FieldSchema> cols,
- boolean applyDistinct, String filter, boolean ascending,
- List<FieldSchema> order, long maxParts) throws MetaException {
+ public PartitionValuesResponse listPartitionValues(String catName, String dbName,
+ String tableName, List<FieldSchema> cols,
+ boolean applyDistinct, String filter,
+ boolean ascending, List<FieldSchema> order,
+ long maxParts) throws MetaException {
+ catName = normalizeIdentifier(catName);
dbName = dbName.toLowerCase().trim();
tableName = tableName.toLowerCase().trim();
try {
if (filter == null || filter.isEmpty()) {
- PartitionValuesResponse response =
- getDistinctValuesForPartitionsNoTxn(dbName, tableName, cols, applyDistinct, ascending, maxParts);
+ PartitionValuesResponse response = getDistinctValuesForPartitionsNoTxn(catName, dbName,
+ tableName, cols, applyDistinct, maxParts);
LOG.info("Number of records fetched: {}", response.getPartitionValues().size());
return response;
} else {
PartitionValuesResponse response =
- extractPartitionNamesByFilter(dbName, tableName, filter, cols, ascending, applyDistinct, maxParts);
+ extractPartitionNamesByFilter(catName, dbName, tableName, filter, cols, ascending, maxParts);
if (response != null && response.getPartitionValues() != null) {
LOG.info("Number of records fetched with filter: {}", response.getPartitionValues().size());
}
@@ -2607,31 +2793,33 @@ public class ObjectStore implements RawStore, Configurable {
} catch (Exception t) {
LOG.error("Exception in ORM", t);
throw new MetaException("Error retrieving partition values: " + t);
- } finally {
}
}
- private PartitionValuesResponse extractPartitionNamesByFilter(String dbName, String tableName, String filter,
- List<FieldSchema> cols, boolean ascending, boolean applyDistinct, long maxParts)
+ private PartitionValuesResponse extractPartitionNamesByFilter(
+ String catName, String dbName, String tableName, String filter, List<FieldSchema> cols,
+ boolean ascending, long maxParts)
throws MetaException, NoSuchObjectException {
- LOG.info("Database: {} Table: {} filter: \"{}\" cols: {}", dbName, tableName, filter, cols);
+ LOG.info("Table: {} filter: \"{}\" cols: {}",
+ getCatalogQualifiedTableName(catName, dbName, tableName), filter, cols);
List<String> partitionNames = null;
List<Partition> partitions = null;
- Table tbl = getTable(dbName, tableName);
+ Table tbl = getTable(catName, dbName, tableName);
try {
// Get partitions by name - ascending or descending
- partitionNames = getPartitionNamesByFilter(dbName, tableName, filter, ascending, maxParts);
+ partitionNames = getPartitionNamesByFilter(catName, dbName, tableName, filter, ascending,
+ maxParts);
} catch (MetaException e) {
LOG.warn("Querying by partition names failed, trying out with partition objects, filter: {}", filter);
}
if (partitionNames == null) {
- partitions = getPartitionsByFilter(dbName, tableName, filter, (short) maxParts);
+ partitions = getPartitionsByFilter(catName, dbName, tableName, filter, (short) maxParts);
}
if (partitions != null) {
- partitionNames = new ArrayList<String>(partitions.size());
+ partitionNames = new ArrayList<>(partitions.size());
for (Partition partition : partitions) {
// Check for NULL's just to be safe
if (tbl.getPartitionKeys() != null && partition.getValues() != null) {
@@ -2642,7 +2830,7 @@ public class ObjectStore implements RawStore, Configurable {
if (partitionNames == null && partitions == null) {
throw new MetaException("Cannot obtain list of partitions by filter:\"" + filter +
- "\" for " + dbName + ":" + tableName);
+ "\" for " + getCatalogQualifiedTableName(catName, dbName, tableName));
}
if (!ascending) {
@@ -2651,7 +2839,7 @@ public class ObjectStore implements RawStore, Configurable {
// Return proper response
PartitionValuesResponse response = new PartitionValuesResponse();
- response.setPartitionValues(new ArrayList<PartitionValuesRow>(partitionNames.size()));
+ response.setPartitionValues(new ArrayList<>(partitionNames.size()));
LOG.info("Converting responses to Partition values for items: {}", partitionNames.size());
for (String partName : partitionNames) {
ArrayList<String> vals = new ArrayList<String>(Collections.nCopies(tbl.getPartitionKeys().size(), null));
@@ -2665,26 +2853,27 @@ public class ObjectStore implements RawStore, Configurable {
return response;
}
- private List<String> getPartitionNamesByFilter(String dbName, String tableName,
+ private List<String> getPartitionNamesByFilter(String catName, String dbName, String tableName,
String filter, boolean ascending, long maxParts)
throws MetaException {
boolean success = false;
- List<String> partNames = new ArrayList<String>();
+ List<String> partNames = new ArrayList<>();
try {
openTransaction();
LOG.debug("Executing getPartitionNamesByFilter");
+ catName = normalizeIdentifier(catName);
dbName = dbName.toLowerCase();
tableName = tableName.toLowerCase();
- MTable mtable = getMTable(dbName, tableName);
+ MTable mtable = getMTable(catName, dbName, tableName);
if( mtable == null ) {
// To be consistent with the behavior of listPartitionNames, if the
// table or db does not exist, we return an empty list
return partNames;
}
- Map<String, Object> params = new HashMap<String, Object>();
- String queryFilterString = makeQueryFilterString(dbName, mtable, filter, params);
+ Map<String, Object> params = new HashMap<>();
+ String queryFilterString = makeQueryFilterString(catName, dbName, mtable, filter, params);
Query query = pm.newQuery(
"select partitionName from org.apache.hadoop.hive.metastore.model.MPartition "
+ "where " + queryFilterString);
@@ -2723,15 +2912,16 @@ public class ObjectStore implements RawStore, Configurable {
return partNames;
}
- private PartitionValuesResponse getDistinctValuesForPartitionsNoTxn(String dbName, String tableName, List<FieldSchema> cols,
- boolean applyDistinct, boolean ascending, long maxParts)
+ private PartitionValuesResponse getDistinctValuesForPartitionsNoTxn(
+ String catName, String dbName, String tableName, List<FieldSchema> cols,
+ boolean applyDistinct, long maxParts)
throws MetaException {
-
try {
openTransaction();
Query q = pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition "
- + "where table.database.name == t1 && table.tableName == t2 ");
- q.declareParameters("java.lang.String t1, java.lang.String t2");
+ + "where table.database.name == t1 && table.database.catalogName == t2 && " +
+ "table.tableName == t3 ");
+ q.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
// TODO: Ordering seems to affect the distinctness, needs checking, disabling.
/*
@@ -2748,7 +2938,7 @@ public class ObjectStore implements RawStore, Configurable {
if (applyDistinct) {
partValuesSelect.append("DISTINCT ");
}
- List<FieldSchema> partitionKeys = getTable(dbName, tableName).getPartitionKeys();
+ List<FieldSchema> partitionKeys = getTable(catName, dbName, tableName).getPartitionKeys();
for (FieldSchema key : cols) {
partValuesSelect.append(extractPartitionKey(key, partitionKeys)).append(", ");
}
@@ -2757,9 +2947,9 @@ public class ObjectStore implements RawStore, Configurable {
q.setResult(partValuesSelect.toString());
PartitionValuesResponse response = new PartitionValuesResponse();
- response.setPartitionValues(new ArrayList<PartitionValuesRow>());
+ response.setPartitionValues(new ArrayList<>());
if (cols.size() > 1) {
- List<Object[]> results = (List<Object[]>) q.execute(dbName, tableName);
+ List<Object[]> results = (List<Object[]>) q.execute(dbName, catName, tableName);
for (Object[] row : results) {
PartitionValuesRow rowResponse = new PartitionValuesRow();
for (Object columnValue : row) {
@@ -2768,7 +2958,7 @@ public class ObjectStore implements RawStore, Configurable {
response.addToPartitionValues(rowResponse);
}
} else {
- List<Object> results = (List<Object>) q.execute(dbName, tableName);
+ List<Object> results = (List<Object>) q.execute(dbName, catName, tableName);
for (Object row : results) {
PartitionValuesRow rowResponse = new PartitionValuesRow();
rowResponse.addToRow((String) row);
@@ -2782,24 +2972,25 @@ public class ObjectStore implements RawStore, Configurable {
}
}
- private List<String> getPartitionNamesNoTxn(String dbName, String tableName, short max) {
+ private List<String> getPartitionNamesNoTxn(String catName, String dbName, String tableName, short max) {
List<String> pns = new ArrayList<>();
if (max == 0) {
return pns;
}
+ catName = normalizeIdentifier(catName);
dbName = normalizeIdentifier(dbName);
tableName = normalizeIdentifier(tableName);
Query query =
pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition "
- + "where table.database.name == t1 && table.tableName == t2 "
+ + "where table.database.name == t1 && table.tableName == t2 && table.database.catalogName == t3 "
+ "order by partitionName asc");
- query.declareParameters("java.lang.String t1, java.lang.String t2");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
query.setResult("partitionName");
if (max > 0) {
query.setRange(0, max);
}
- Collection<String> names = (Collection<String>) query.execute(dbName, tableName);
+ Collection<String> names = (Collection<String>) query.execute(dbName, tableName, catName);
pns.addAll(names);
if (query != null) {
@@ -2823,14 +3014,16 @@ public class ObjectStore implements RawStore, Configurable {
* you want results for. E.g., if resultsCol is partitionName, the Collection
* has types of String, and if resultsCol is null, the types are MPartition.
*/
- private Collection getPartitionPsQueryResults(String dbName, String tableName,
+ private Collection getPartitionPsQueryResults(String catName, String dbName, String tableName,
List<String> part_vals, short max_parts, String resultsCol, QueryWrapper queryWrapper)
throws MetaException, NoSuchObjectException {
+ catName = normalizeIdentifier(catName);
dbName = normalizeIdentifier(dbName);
tableName = normalizeIdentifier(tableName);
- Table table = getTable(dbName, tableName);
+ Table table = getTable(catName, dbName, tableName);
if (table == null) {
- throw new NoSuchObjectException(dbName + "." + tableName + " table not found");
+ throw new NoSuchObjectException(getCatalogQualifiedTableName(catName, dbName, tableName)
+ + " table not found");
}
List<FieldSchema> partCols = table.getPartitionKeys();
int numPartKeys = partCols.size();
@@ -2851,10 +3044,11 @@ public class ObjectStore implements RawStore, Configurable {
}
Query query = queryWrapper.query = pm.newQuery(MPartition.class);
StringBuilder queryFilter = new StringBuilder("table.database.name == dbName");
+ queryFilter.append(" && table.database.catalogName == catName");
queryFilter.append(" && table.tableName == tableName");
queryFilter.append(" && partitionName.matches(partialRegex)");
query.setFilter(queryFilter.toString());
- query.declareParameters("java.lang.String dbName, "
+ query.declareParameters("java.lang.String dbName, java.lang.String catName, "
+ "java.lang.String tableName, java.lang.String partialRegex");
if (max_parts >= 0) {
// User specified a row limit, set it on the Query
@@ -2864,11 +3058,11 @@ public class ObjectStore implements RawStore, Configurable {
query.setResult(resultsCol);
}
- return (Collection) query.execute(dbName, tableName, partNameMatcher);
+ return (Collection) query.executeWithArray(dbName, catName, tableName, partNameMatcher);
}
@Override
- public List<Partition> listPartitionsPsWithAuth(String db_name, String tbl_name,
+ public List<Partition> listPartitionsPsWithAuth(String catName, String db_name, String tbl_name,
List<String> part_vals, short max_parts, String userName, List<String> groupNames)
throws MetaException, InvalidObjectException, NoSuchObjectException {
List<Partition> partitions = new ArrayList<>();
@@ -2878,9 +3072,9 @@ public class ObjectStore implements RawStore, Configurable {
try {
openTransaction();
LOG.debug("executing listPartitionNamesPsWithAuth");
- Collection parts = getPartitionPsQueryResults(db_name, tbl_name,
+ Collection parts = getPartitionPsQueryResults(catName, db_name, tbl_name,
part_vals, max_parts, null, queryWrapper);
- MTable mtbl = getMTable(db_name, tbl_name);
+ MTable mtbl = getMTable(catName, db_name, tbl_name);
for (Object o : parts) {
Partition part = convertToPart((MPartition) o);
//set auth privileges
@@ -2888,7 +3082,7 @@ public class ObjectStore implements RawStore, Configurable {
"TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
String partName = Warehouse.makePartName(this.convertToFieldSchemas(mtbl
.getPartitionKeys()), part.getValues());
- PrincipalPrivilegeSet partAuth = getPartitionPrivilegeSet(db_name,
+ PrincipalPrivilegeSet partAuth = getPartitionPrivilegeSet(catName, db_name,
tbl_name, partName, userName, groupNames);
part.setPrivileges(partAuth);
}
@@ -2902,7 +3096,7 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<String> listPartitionNamesPs(String dbName, String tableName,
+ public List<String> listPartitionNamesPs(String catName, String dbName, String tableName,
List<String> part_vals, short max_parts) throws MetaException, NoSuchObjectException {
List<String> partitionNames = new ArrayList<>();
boolean success = false;
@@ -2911,7 +3105,7 @@ public class ObjectStore implements RawStore, Configurable {
try {
openTransaction();
LOG.debug("Executing listPartitionNamesPs");
- Collection<String> names = getPartitionPsQueryResults(dbName, tableName,
+ Collection<String> names = getPartitionPsQueryResults(catName, dbName, tableName,
part_vals, max_parts, "partitionName", queryWrapper);
partitionNames.addAll(names);
success = commitTransaction();
@@ -2922,7 +3116,8 @@ public class ObjectStore implements RawStore, Configurable {
}
// TODO:pc implement max
- private List<MPartition> listMPartitions(String dbName, String tableName, int max, QueryWrapper queryWrapper) {
+ private List<MPartition> listMPartitions(String catName, String dbName, String tableName,
+ int max, QueryWrapper queryWrapper) {
boolean success = false;
List<MPartition> mparts = null;
try {
@@ -2930,13 +3125,14 @@ public class ObjectStore implements RawStore, Configurable {
LOG.debug("Executing listMPartitions");
dbName = normalizeIdentifier(dbName);
tableName = normalizeIdentifier(tableName);
- Query query = queryWrapper.query = pm.newQuery(MPartition.class, "table.tableName == t1 && table.database.name == t2");
- query.declareParameters("java.lang.String t1, java.lang.String t2");
+ Query query = queryWrapper.query = pm.newQuery(MPartition.class,
+ "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3");
+ query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3");
query.setOrdering("partitionName ascending");
if (max > 0) {
query.setRange(0, max);
}
- mparts = (List<MPartition>) query.execute(tableName, dbName);
+ mparts = (List<MPartition>) query.execute(tableName, dbName, catName);
LOG.debug("Done executing query for listMPartitions");
pm.retrieveAll(mparts);
success = commitTransaction();
@@ -2950,41 +3146,43 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public List<Partition> getPartitionsByNames(String dbName, String tblName,
+ public List<Partition> getPartitionsByNames(String catName, String dbName, String tblName,
List<String> partNames) throws MetaException, NoSuchObjectException {
- return getPartitionsByNamesInternal(dbName, tblName, partNames, true, true);
+ return getPartitionsByNamesInternal(catName, dbName, tblName, partNames, true, true);
}
- protected List<Partition> getPartitionsByNamesInternal(String dbName, String tblName,
- final List<String> partNames, boolean allowSql, boolean allowJdo)
+ protected List<Partition> getPartitionsByNamesInternal(String catName, String dbName,
+ String tblName,
+ final List<String> partNames,
+ boolean allowSql, boolean allowJdo)
throws MetaException, NoSuchObjectException {
- return new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) {
+ return new GetListHelper<Partition>(catName, dbName, tblName, allowSql, allowJdo) {
@Override
protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException {
- return directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames);
+ return directSql.getPartitionsViaSqlFilter(catName, dbName, tblName, partNames);
}
@Override
protected List<Partition> getJdoResult(
GetHelper<List<Partition>> ctx) throws MetaException, NoSuchObjectException {
- return getPartitionsViaOrmFilter(dbName, tblName, partNames);
+ return getPartitionsViaOrmFilter(catName, dbName, tblName, partNames);
}
}.run(false);
}
@Override
- public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr,
+ public boolean getPartitionsByExpr(String catName, String dbName, String tblName, byte[] expr,
String defaultPartitionName, short maxParts, List<Partition> result) throws TException {
return getPartitionsByExprInternal(
- dbName, tblName, expr, defaultPartitionName, maxParts, result, true, true);
+ catName, dbName, tblName, expr, defaultPartitionName, maxParts, result, true, true);
}
- protected boolean getPartitionsByExprInternal(String dbName, String tblName, final byte[] expr,
+ protected boolean getPartitionsByExprInternal(String catName, String dbName, String tblName, final byte[] expr,
final String defaultPartitionName, final short maxParts, List<Partition> result,
boolean allowSql, boolean allowJdo) throws TException {
assert result != null;
final ExpressionTree exprTree = PartFilterExprUtil.makeExpressionTree(expressionProxy, expr);
final AtomicBoolean hasUnknownPartitions = new AtomicBoolean(false);
- result.addAll(new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) {
+ result.addAll(new GetListHelper<Partition>(catName, dbName, tblName, allowSql, allowJdo) {
@Override
protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException {
// If we have some sort of expression tree, try SQL filter pushdown.
@@ -2999,7 +3197,7 @@ public class ObjectStore implements RawStore, Configurable {
List<String> partNames = new LinkedList<>();
hasUnknownPartitions.set(getPartitionNamesPrunedByExprNoTxn(
ctx.getTable(), expr, defaultPartitionName, maxParts, partNames));
- return directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames);
+ return directSql.getPartitionsViaSqlFilter(catName, dbName, tblName, partNames);
}
@Override
@@ -3015,7 +3213,7 @@ public class ObjectStore implements RawStore, Configurable {
List<String> partNames = new ArrayList<>();
hasUnknownPartitions.set(getPartitionNamesPrunedByExprNoTxn(
ctx.getTable(), expr, defaultPartitionName, maxParts, partNames));
- result = getPartitionsViaOrmFilter(dbName, tblName, partNames);
+ result = getPartitionsViaOrmFilter(catName, dbName, tblName, partNames);
}
return result;
}
@@ -3036,7 +3234,7 @@ public class ObjectStore implements RawStore, Configurable {
*/
private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr,
String defaultPartName, short maxParts, List<String> result) throws MetaException {
- result.addAll(getPartitionNamesNoTxn(
+ result.addAll(getPartitionNamesNoTxn(table.getCatName(),
table.getDbName(), table.getTableName(), maxParts));
if (defaultPartName == null || defaultPartName.isEmpty()) {
defaultPartName = MetastoreConf.getVar(getConf(), ConfVars.DEFAULTPARTITIONNAME);
@@ -3058,7 +3256,7 @@ public class ObjectStore implements RawStore, Configurable {
short maxParts, boolean isValidatedFilter) throws MetaException {
Map<String, Object> params = new HashMap<>();
String jdoFilter =
- makeQueryFilterString(table.getDbName(), table, tree, params, isValidatedFilter);
+ makeQueryFilterString(table.getCatName(), table.getDbName(), table, tree, params, isValidatedFilter);
if (jdoFilter == null) {
assert !isValidatedFilter;
return null;
@@ -3085,7 +3283,8 @@ public class ObjectStore implements RawStore, Configurable {
private Integer getNumPartitionsViaOrmFilter(Table table, ExpressionTree tree, boolean isValidatedFilter)
throws MetaException {
Map<String, Object> params = new HashMap<>();
- String jdoFilter = makeQueryFilterString(table.getDbName(), table, tree, params, isValidatedFilter);
+ String jdoFilter = makeQueryFilterString(table.getCatName(), table.getDbName(), table, tree,
+ params, isValidatedFilter);
if (jdoFilter == null) {
assert !isValidatedFilter;
return null;
@@ -3110,29 +3309,29 @@ public class ObjectStore implements RawStore, Configurable {
* @param partNames Partition names to get the objects for.
* @return Resulting partitions.
*/
- private List<Partition> getPartitionsViaOrmFilter(
+ private List<Partition> getPartitionsViaOrmFilter(String catName,
String dbName, String tblName, List<String> partNames) throws MetaException {
if (partNames.isEmpty()) {
return new ArrayList<>();
}
ObjectPair<Query, Map<String, String>> queryWithParams =
- getPartQueryWithParams(dbName, tblName, partNames);
+ getPartQueryWithParams(catName, dbName, tblName, partNames);
Query query = queryWithParams.getFirst();
query.setResultClass(MPartition.class);
query.setClass(MPartition.class);
query.setOrdering("partitionName ascending");
@SuppressWarnings("unchecked")
List<MPartition> mparts = (List<MPartition>)query.executeWithMap(queryWithParams.getSecond());
- List<Partition> partitions = convertToParts(dbName, tblName, mparts);
+ List<Partition> partitions = convertToParts(catName, dbName, tblName, mparts);
if (query != null) {
query.closeAll();
}
return partitions;
}
- private void dropPartitionsNoTxn(String dbName, String tblName, List<String> partNames) {
+ private void dropPartitionsNoTxn(String catName, String dbName, String tblName, List<String> partNames) {
ObjectPair<Query, Map<String, String>> queryWithParams =
- getPartQueryWithParams(dbName, tblName, partNames);
+ getPartQueryWithParams(catName, dbName, tblName, partNames);
Query query = queryWithParams.getFirst();
query.setClass(MPartition.class);
long deleted = query.deletePersistentAll(queryWithParams.getSecond());
@@ -3147,9 +3346,9 @@ public class ObjectStore implements RawStore, Configurable {
* they are referenced by other SDs.
*/
private HashSet<MColumnDescriptor> detachCdsFromSdsNoTxn(
- String dbName, String tblName, List<String> partNames) {
+ String catName, String dbName, String tblName, List<String> partNames) {
ObjectPair<Query, Map<String, String>> queryWithParams =
- getPartQueryWithParams(dbName, tblName, partNames);
+ getPartQueryWithParams(catName, dbName, tblName, partNames);
Query query = queryWithParams.getFirst();
query.setClass(MPartition.class);
query.setResult("sd");
@@ -3169,9 +3368,10 @@ public class ObjectStore implements RawStore, Configurable {
return candidateCds;
}
- private ObjectPair<Query, Map<String, String>> getPartQueryWithParams(String dbName,
- String tblName, List<String> partNames) {
- StringBuilder sb = new StringBuilder("table.tableName == t1 && table.database.name == t2 && (");
+ private ObjectPair<Query, Map<String, String>> getPartQueryWithParams(
+ String catName, String dbName, String tblName, List<String> partNames) {
+ StringBuilder sb = new StringBuilder("table.tableName == t1 && table.database.name == t2 &&" +
+ " table.database.catalogName == t3 && (");
int n = 0;
Map<String, String> params = new HashMap<>();
for (Iterator<String> itr = partNames.iterator(); itr.hasNext();) {
@@ -3189,14 +3389,15 @@ public class ObjectStore implements RawStore, Configurable {
LOG.debug(" JDOQL filter is {}", sb);
params.put("t1", normalizeIdentifier(tblName));
params.put("t2", normalizeIdentifier(dbName));
+ params.put("t3", normalizeIdentifier(catName));
query.declareParameters(makeParameterDeclarationString(params));
return new ObjectPair<>(query, params);
}
@Override
- public List<Partition> getPartitionsByFilter(String dbName, String tblName,
+ public List<Partition> getPartitionsByFilter(String catName, String dbName, String tblName,
String filter, short maxParts) throws MetaException, NoSuchObjectException {
- return getPartitionsByFilterInternal(dbName, tblName, filter, maxParts, true, true);
+ return getPartitionsByFilterInternal(catName, dbName, tblName, filter, maxParts, true, true);
}
/** Helper class for getting stuff w/transaction, direct SQL, perf logging, etc. */
@@ -3206,14 +3407,15 @@ public class ObjectStore implements RawStore, Configurable {
private boolean doUseDirectSql;
private long start;
private Table table;
- protected final String dbName, tblName;
+ protected final String catName, dbName, tblName;
private boolean success = false;
protected T results = null;
- public GetHelper(String dbName, String tblName, boolean allowSql, boolean allowJdo)
- throws MetaException {
+ public GetHelper(String catalogName, String dbName, String tblName,
+ boolean allowSql, boolean allowJdo) throws MetaException {
assert allowSql || allowJdo;
this.allowJdo = allowJdo;
+ this.catName = normalizeIdentifier(catalogName);
this.dbName = normalizeIdentifier(dbName);
if (tblName != null){
this.tblName = normalizeIdentifier(tblName);
@@ -3284,7 +3486,7 @@ public class ObjectStore implements RawStore, Configurable {
start = doTrace ? System.nanoTime() : 0;
openTransaction();
if (initTable && (tblName != null)) {
- table = ensureGetTable(dbName, tblName);
+ table = ensureGetTable(catName, dbName, tblName);
}
doUseDirectSql = doUseDirectSql && canUseDirectSql(this);
}
@@ -3324,7 +3526,7 @@ public class ObjectStore implements RawStore, Configurable {
start = doTrace ? System.nanoTime() : 0;
openTransaction();
if (table != null) {
- table = ensureGetTable(dbName, tblName);
+ table = ensureGetTable(catName, dbName, tblName);
}
} else {
start = doTrace ? System.nanoTime() : 0;
@@ -3393,9 +3595,9 @@ public class ObjectStore implements RawStore, Configurable {
}
private abstract class GetListHelper<T> extends GetHelper<List<T>> {
- public GetListHelper(
- String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException {
- super(dbName, tblName, allowSql, allowJdo);
+ public GetListHelper(String catName, String dbName, String tblName, boolean allowSql,
+ boolean allowJdo) throws MetaException {
+ super(catName, dbName, tblName, allowSql, allowJdo);
}
@Override
@@ -3413,9 +3615,9 @@ public class ObjectStore implements RawStore, Configurable {
* @param allowJdo Whether or not we allow ORM to perform this query.
* @throws MetaException
*/
- public GetDbHelper(
- String dbName,boolean allowSql, boolean allowJdo) throws MetaException {
- super(dbName,null,allowSql,allowJdo);
+ public GetDbHelper(String catalogName, String dbName,boolean allowSql, boolean allowJdo)
+ throws MetaException {
+ super(catalogName, dbName,null,allowSql,allowJdo);
}
@Override
@@ -3425,9 +3627,9 @@ public class ObjectStore implements RawStore, Configurable {
}
private abstract class GetStatHelper extends GetHelper<ColumnStatistics> {
- public GetStatHelper(
- String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException {
- super(dbName, tblName, allowSql, allowJdo);
+ public GetStatHelper(String catalogName, String dbName, String tblName, boolean allowSql,
+ boolean allowJdo) throws MetaException {
+ super(catalogName, dbName, tblName, allowSql, allowJdo);
}
@Override
@@ -3437,12 +3639,12 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public int getNumPartitionsByFilter(String dbName, String tblName,
+ public int getNumPartitionsByFilter(String catName, String dbName, String tblName,
String filter) throws MetaException, NoSuchObjectException {
final ExpressionTree exprTree = org.apache.commons.lang
<TRUNCATED>