You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2018/06/15 21:37:51 UTC
hive git commit: HIVE-19861: Fix temp table path generation for acid
table export (Jason Dere, reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/master c93a15553 -> bceb3dd82
HIVE-19861: Fix temp table path generation for acid table export (Jason Dere, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bceb3dd8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bceb3dd8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bceb3dd8
Branch: refs/heads/master
Commit: bceb3dd82cbeb0334c3c33da7a21a74ad631810b
Parents: c93a155
Author: Jason Dere <jd...@hortonworks.com>
Authored: Fri Jun 15 14:36:42 2018 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Fri Jun 15 14:36:42 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 22 ++++++++++++--------
.../ql/metadata/SessionHiveMetaStoreClient.java | 5 +++++
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 4 +---
.../hadoop/hive/ql/plan/CreateTableDesc.java | 2 +-
.../hadoop/hive/ql/session/SessionState.java | 8 +++++++
5 files changed, 28 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bceb3dd8/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index e069499..46a9f7d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -5090,7 +5090,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
// If location is specified - ensure that it is a full qualified name
if (DDLTask.doesTableNeedLocation(tbl)) {
- makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tbl.getTableName(), conf);
+ makeLocationQualified(tbl.getDbName(), tbl, conf);
}
if (crtTbl.getLocation() == null && !tbl.isPartitioned()
@@ -5267,17 +5267,21 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
* @param name
* Object name.
*/
- public static void makeLocationQualified(String databaseName, StorageDescriptor sd,
- String name, HiveConf conf) throws HiveException {
+ public static void makeLocationQualified(String databaseName, Table table, HiveConf conf) throws HiveException {
Path path = null;
+ StorageDescriptor sd = table.getTTable().getSd();
+ String name = table.getTableName();
if (!sd.isSetLocation())
{
- // Location is not set, leave it as-is if this is not a default DB
- if (databaseName.equalsIgnoreCase(Warehouse.DEFAULT_DATABASE_NAME))
- {
- // Default database name path is always ignored, use METASTOREWAREHOUSE and object name
- // instead
- path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE), org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(name.toLowerCase()));
+ // Leave temp tables as-is, default location will be handled by SessionHiveMetastoreClient.
+ if (!table.isTemporary()) {
+ // Location is not set, leave it as-is if this is not a default DB
+ if (databaseName.equalsIgnoreCase(Warehouse.DEFAULT_DATABASE_NAME))
+ {
+ // Default database name path is always ignored, use METASTOREWAREHOUSE and object name
+ // instead
+ path = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE), org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(name.toLowerCase()));
+ }
}
}
else
http://git-wip-us.apache.org/repos/asf/hive/blob/bceb3dd8/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 209fdfb..58c8960 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -456,6 +456,11 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
// Create temp table directory
Warehouse wh = getWh();
+ if (tbl.getSd().getLocation() == null) {
+ // Temp tables that do not go through SemanticAnalyzer may not have location set - do it here.
+ // For example export of acid tables generates a query plan that creates a temp table.
+ tbl.getSd().setLocation(SessionState.generateTempTableLocation(conf));
+ }
Path tblPath = wh.getDnsPath(new Path(tbl.getSd().getLocation()));
if (tblPath == null) {
throw new MetaException("Temp table path not set for " + tbl.getTableName());
http://git-wip-us.apache.org/repos/asf/hive/blob/bceb3dd8/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 2e055ab..00957ea 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -13144,9 +13144,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
try {
// Generate a unique ID for temp table path.
// This path will be fixed for the life of the temp table.
- Path path = new Path(SessionState.getTempTableSpace(conf), UUID.randomUUID().toString());
- path = Warehouse.getDnsPath(path, conf);
- location = path.toString();
+ location = SessionState.generateTempTableLocation(conf);
} catch (MetaException err) {
throw new SemanticException("Error while generating temp table path:", err);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/bceb3dd8/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
index 0429278..871844b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
@@ -822,7 +822,7 @@ public class CreateTableDesc extends DDLDesc implements Serializable {
if (DDLTask.doesTableNeedLocation(tbl)) {
// If location is specified - ensure that it is a full qualified name
- DDLTask.makeLocationQualified(tbl.getDbName(), tbl.getTTable().getSd(), tableName, conf);
+ DDLTask.makeLocationQualified(tbl.getDbName(), tbl, conf);
}
if (isExternal()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/bceb3dd8/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 6ff8f99..81864f5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -59,7 +59,9 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.conf.HiveConfUtil;
import org.apache.hadoop.hive.metastore.ObjectStore;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.cache.CachedStore;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
@@ -844,6 +846,12 @@ public class SessionState {
return this.hdfsTmpTableSpace;
}
+ public static String generateTempTableLocation(Configuration conf) throws MetaException {
+ Path path = new Path(SessionState.getTempTableSpace(conf), UUID.randomUUID().toString());
+ path = Warehouse.getDnsPath(path, conf);
+ return path.toString();
+ }
+
@VisibleForTesting
void releaseSessionLockFile() throws IOException {
if (hdfsSessionPath != null && hdfsSessionPathLockFile != null) {