You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/08/21 20:53:37 UTC
hive git commit: HIVE-20411 : Hive.loadPartition doesn't support
catalogs (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 20baf490c -> 85b04f5fd
HIVE-20411 : Hive.loadPartition doesn't support catalogs (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/85b04f5f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/85b04f5f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/85b04f5f
Branch: refs/heads/master
Commit: 85b04f5fd5b602fa34d0f9756037ebf451dc6a65
Parents: 20baf49
Author: sergey <se...@apache.org>
Authored: Tue Aug 21 13:33:16 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Aug 21 13:33:16 2018 -0700
----------------------------------------------------------------------
.../metastore/SynchronizedMetaStoreClient.java | 4 ++--
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 8 +++++--
.../apache/hadoop/hive/ql/exec/MoveTask.java | 3 ++-
.../ql/hooks/UpdateInputAccessTimeHook.java | 2 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 22 ++++++++++++++------
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 1 +
.../ql/util/HiveStrictManagedMigration.java | 15 ++++++-------
.../hive/metastore/HiveMetaStoreClient.java | 3 ++-
.../hadoop/hive/metastore/IMetaStoreClient.java | 3 ++-
.../HiveMetaStoreClientPreCatalog.java | 2 +-
10 files changed, 41 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/ql/src/java/org/apache/hadoop/hive/metastore/SynchronizedMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/metastore/SynchronizedMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/metastore/SynchronizedMetaStoreClient.java
index 1e279f3..0ab77e8 100644
--- a/ql/src/java/org/apache/hadoop/hive/metastore/SynchronizedMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/metastore/SynchronizedMetaStoreClient.java
@@ -78,9 +78,9 @@ public final class SynchronizedMetaStoreClient {
return client.add_partition(partition);
}
- public synchronized void alter_partition(String dbName, String tblName,
+ public synchronized void alter_partition(String catName, String dbName, String tblName,
Partition newPart, EnvironmentContext environmentContext, String writeIdList) throws TException {
- client.alter_partition(dbName, tblName, newPart, environmentContext, writeIdList);
+ client.alter_partition(catName, dbName, tblName, newPart, environmentContext, writeIdList);
}
public synchronized LockResponse checkLock(long lockid) throws TException {
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 467f728..58c3ae1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -1435,7 +1435,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
*/
private int touch(Hive db, AlterTableSimpleDesc touchDesc)
throws HiveException {
-
+ // TODO: catalog
Table tbl = db.getTable(touchDesc.getTableName());
EnvironmentContext environmentContext = new EnvironmentContext();
environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
@@ -1450,7 +1450,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
throw new HiveException("Specified partition does not exist");
}
try {
- db.alterPartition(touchDesc.getTableName(), part, environmentContext, true);
+ db.alterPartition(tbl.getCatalogName(), tbl.getDbName(), tbl.getTableName(),
+ part, environmentContext, true);
} catch (InvalidOperationException e) {
throw new HiveException(e);
}
@@ -1799,6 +1800,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
authority.toString(),
harPartitionDir.getPath()); // make in Path to ensure no slash at the end
setArchived(p, harPath, partSpecInfo.values.size());
+ // TODO: catalog
db.alterPartition(simpleDesc.getTableName(), p, null, true);
}
} catch (Exception e) {
@@ -2005,6 +2007,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
for(Partition p: partitions) {
setUnArchived(p);
try {
+ // TODO: catalog
db.alterPartition(simpleDesc.getTableName(), p, null, true);
} catch (InvalidOperationException e) {
throw new HiveException(e);
@@ -4766,6 +4769,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
// drop the table
+ // TODO: API w/catalog name
db.dropTable(dropTbl.getTableName(), dropTbl.getIfPurge());
if (tbl != null) {
// Remove from cache if it is a materialized view
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index d2c04e2..827721f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -798,7 +798,8 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
}
if (updateBucketCols || updateSortCols) {
- db.alterPartition(table.getDbName(), table.getTableName(), partn, null, true);
+ db.alterPartition(table.getCatalogName(), table.getDbName(), table.getTableName(),
+ partn, null, true);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java
index ea0b2c3..36a92aa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java
@@ -73,7 +73,7 @@ public class UpdateInputAccessTimeHook {
Table t = db.getTable(dbName, tblName);
p = db.getPartition(t, p.getSpec(), false);
p.setLastAccessTime(lastAccessTime);
- db.alterPartition(dbName, tblName, p, null, false);
+ db.alterPartition(null, dbName, tblName, p, null, false);
t.setLastAccessTime(lastAccessTime);
db.alterTable(dbName + "." + tblName, t, false, null, false);
break;
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 9b82080..36dc694 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -682,11 +682,12 @@ public class Hive {
* if the changes in metadata is not acceptable
* @throws TException
*/
+ @Deprecated
public void alterPartition(String tblName, Partition newPart,
EnvironmentContext environmentContext, boolean transactional)
throws InvalidOperationException, HiveException {
String[] names = Utilities.getDbTableName(tblName);
- alterPartition(names[0], names[1], newPart, environmentContext, transactional);
+ alterPartition(null, names[0], names[1], newPart, environmentContext, transactional);
}
/**
@@ -706,10 +707,13 @@ public class Hive {
* if the changes in metadata is not acceptable
* @throws TException
*/
- public void alterPartition(String dbName, String tblName, Partition newPart,
+ public void alterPartition(String catName, String dbName, String tblName, Partition newPart,
EnvironmentContext environmentContext, boolean transactional)
throws InvalidOperationException, HiveException {
try {
+ if (catName == null) {
+ catName = getDefaultCatalog(conf);
+ }
validatePartition(newPart);
String location = newPart.getLocation();
if (location != null) {
@@ -728,7 +732,7 @@ public class Hive {
LOG.warn("Cannot get a table snapshot for " + tblName);
}
}
- getSynchronizedMSC().alter_partition(
+ getSynchronizedMSC().alter_partition(catName,
dbName, tblName, newPart.getTPartition(), environmentContext,
tableSnapshot == null ? null : tableSnapshot.getValidWriteIdList());
@@ -849,6 +853,7 @@ public class Hive {
}
}
+ // TODO: this whole path won't work with catalogs
public void alterDatabase(String dbName, Database db)
throws HiveException {
try {
@@ -872,6 +877,8 @@ public class Hive {
createTable(tbl, false);
}
+ // TODO: from here down dozens of methods do not support catalog. I got tired marking them.
+
/**
* Creates the table with the given objects. It takes additional arguments for
* primary keys and foreign keys associated with the table.
@@ -1071,12 +1078,12 @@ public class Hive {
public void truncateTable(String dbDotTableName, Map<String, String> partSpec) throws HiveException {
try {
Table table = getTable(dbDotTableName, true);
- // TODO: we should refactor code to make sure snapshot is always obtained in the same layer e.g. Hive.java
AcidUtils.TableSnapshot snapshot = null;
if (AcidUtils.isTransactionalTable(table)) {
snapshot = AcidUtils.getTableSnapshot(conf, table, true);
}
+ // TODO: APIs with catalog names
List<String> partNames = ((null == partSpec)
? null : getPartitionNames(table.getDbName(), table.getTableName(), partSpec, (short) -1));
if (snapshot == null) {
@@ -1130,6 +1137,7 @@ public class Hive {
* if there's an internal error or if the table doesn't exist
*/
public Table getTable(final String dbName, final String tableName) throws HiveException {
+ // TODO: catalog... etc everywhere
if (tableName.contains(".")) {
String[] names = Utilities.getDbTableName(tableName);
return this.getTable(names[0], names[1], true);
@@ -2117,7 +2125,7 @@ public class Hive {
ec.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
}
LOG.debug("Altering existing partition " + newTPart.getSpec());
- getSynchronizedMSC().alter_partition(
+ getSynchronizedMSC().alter_partition(tbl.getCatName(),
tbl.getDbName(), tbl.getTableName(), newTPart.getTPartition(), new EnvironmentContext(),
tableSnapshot == null ? null : tableSnapshot.getValidWriteIdList());
}
@@ -2591,6 +2599,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
public List<Partition> createPartitions(AddPartitionDesc addPartitionDesc) throws HiveException {
+ // TODO: catalog name everywhere in this method
Table tbl = getTable(addPartitionDesc.getDbName(), addPartitionDesc.getTableName());
int size = addPartitionDesc.getPartitionCount();
List<org.apache.hadoop.hive.metastore.api.Partition> in =
@@ -2799,7 +2808,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
if (!org.apache.commons.lang.StringUtils.isEmpty(tbl.getDbName())) {
fullName = tbl.getFullyQualifiedName();
}
- alterPartition(fullName, new Partition(tbl, tpart), null, true);
+ alterPartition(tbl.getCatalogName(), tbl.getDbName(), tbl.getTableName(),
+ new Partition(tbl, tpart), null, true);
}
private void alterPartitionSpecInMemory(Table tbl,
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
index 0d80ed3..e7a33f5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
@@ -271,6 +271,7 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
if(ctx.isExplainPlan()) {
try {
//so that "explain" doesn't "leak" tmp tables
+ // TODO: catalog
db.dropTable(newTable.getDbName(), newTable.getTableName(), true, true, true);
} catch(HiveException ex) {
LOG.warn("Unable to drop " + newTableName + " due to: " + ex.getMessage(), ex);
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
index 2a737bb..806464c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
@@ -241,7 +241,7 @@ public class HiveStrictManagedMigration {
}
String oldWarehouseRoot = cli.getOptionValue("oldWarehouseRoot");
boolean dryRun = cli.hasOption("dryRun");
-
+
RunOptions runOpts = new RunOptions(
dbRegex,
tableRegex,
@@ -499,7 +499,7 @@ public class HiveStrictManagedMigration {
oldDefaultDbLocation, curWhRootPath, dbName);
}
}
- }
+ }
return false;
}
@@ -576,7 +576,7 @@ public class HiveStrictManagedMigration {
void moveTableData(Database dbObj, Table tableObj, Path newTablePath) throws HiveException, IOException, TException {
String dbName = tableObj.getDbName();
String tableName = tableObj.getTableName();
-
+
Path oldTablePath = new Path(tableObj.getSd().getLocation());
LOG.info("Moving location of {} from {} to {}", getQualifiedName(tableObj), oldTablePath, newTablePath);
@@ -909,8 +909,8 @@ public class HiveStrictManagedMigration {
modifiedTable, false, null, false);
}
- void updatePartitionLocation(String dbName, Table table, String partName, Partition part, Path newLocation)
- throws HiveException, TException {
+ void updatePartitionLocation(String dbName, Table table, String partName,
+ Partition part, Path newLocation) throws HiveException, TException {
String msg = String.format("ALTER TABLE %s PARTITION (%s) SET LOCATION '%s'",
getQualifiedName(table), partName, newLocation.toString());
LOG.info(msg);
@@ -920,7 +920,8 @@ public class HiveStrictManagedMigration {
new org.apache.hadoop.hive.ql.metadata.Table(table),
part);
modifiedPart.setLocation(newLocation.toString());
- hive.alterPartition(dbName, table.getTableName(), modifiedPart, null, false);
+ hive.alterPartition(
+ table.getCatName(), dbName, table.getTableName(), modifiedPart, null, false);
}
void updateTableProperties(Table table, Map<String, String> props) throws HiveException {
@@ -1100,7 +1101,7 @@ public class HiveStrictManagedMigration {
if (isDir && recurse) {
for (FileStatus subFile : fs.listStatus(path)) {
// TODO: Use threadpool for more concurrency?
- // TODO: check/set all files, or only directories
+ // TODO: check/set all files, or only directories
checkAndSetFileOwnerPermissions(fs, subFile, userName, groupName, dirPerms, filePerms, dryRun, recurse);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 5ae00af..faf6810 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2061,11 +2061,12 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
@Override
- public void alter_partition(String dbName, String tblName, Partition newPart,
+ public void alter_partition(String catName, String dbName, String tblName, Partition newPart,
EnvironmentContext environmentContext, String writeIdList)
throws InvalidOperationException, MetaException, TException {
AlterPartitionsRequest req = new AlterPartitionsRequest(
dbName, tblName, Lists.newArrayList(newPart));
+ req.setCatName(catName);
req.setEnvironmentContext(environmentContext);
req.setValidWriteIdList(writeIdList);
client.alter_partitions_req(req);
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
index 91405b9..ac10da2 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
@@ -534,6 +534,7 @@ public interface IMetaStoreClient {
* @throws TException
* A thrift communication error occurred
*/
+ @Deprecated // TODO: deprecate all methods without a catalog here; a single layer (e.g. Hive.java) should handle current-catalog
void dropTable(String dbname, String tableName, boolean deleteData,
boolean ignoreUnknownTab, boolean ifPurge) throws MetaException, TException,
NoSuchObjectException;
@@ -2079,7 +2080,7 @@ public interface IMetaStoreClient {
throws InvalidOperationException, MetaException, TException;
- void alter_partition(String dbName, String tblName, Partition newPart,
+ void alter_partition(String catName, String dbName, String tblName, Partition newPart,
EnvironmentContext environmentContext, String writeIdList)
throws InvalidOperationException, MetaException, TException;
http://git-wip-us.apache.org/repos/asf/hive/blob/85b04f5f/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index 35abd00..6e0d9c1 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -3516,7 +3516,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
@Override
- public void alter_partition(String dbName, String tblName, Partition newPart,
+ public void alter_partition(String catName, String dbName, String tblName, Partition newPart,
EnvironmentContext environmentContext, String writeIdList)
throws InvalidOperationException, MetaException, TException {
throw new UnsupportedOperationException();