You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gs...@apache.org on 2023/04/12 16:33:31 UTC
[hive] branch master updated: HIVE-27164: Create temporary insert only/full acid table fails at metastore transformCreateTable (#4176) (Venugopal Reddy K, reviewed by Sai Hemanth G)
This is an automated email from the ASF dual-hosted git repository.
gsaihemanth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 1f5e9c10ab4 HIVE-27164: Create temporary insert only/full acid table fails at metastore transformCreateTable (#4176) (Venugopal Reddy K, reviewed by Sai Hemanth G)
1f5e9c10ab4 is described below
commit 1f5e9c10ab4d3fbc0f26affa026df021368858a3
Author: Venu Reddy <35...@users.noreply.github.com>
AuthorDate: Wed Apr 12 22:03:18 2023 +0530
HIVE-27164: Create temporary insert only/full acid table fails at metastore transformCreateTable (#4176) (Venugopal Reddy K, reviewed by Sai Hemanth G)
---
.../ql/metadata/SessionHiveMetaStoreClient.java | 10 ++
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 3 +-
.../apache/hadoop/hive/ql/parse/TaskCompiler.java | 4 +-
.../hadoop/hive/ql/metadata/TestTempAcidTable.java | 129 +++++++++++++++++++++
4 files changed, 144 insertions(+), 2 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 07267c8dd41..d16886513a7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -2525,4 +2525,14 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClientWithLocalCach
}
}
+ @Override
+ public org.apache.hadoop.hive.metastore.api.Table getTranslateTableDryrun(
+ org.apache.hadoop.hive.metastore.api.Table tbl) throws AlreadyExistsException,
+ InvalidObjectException, MetaException, NoSuchObjectException, TException {
+ if (tbl.isTemporary()) {
+ org.apache.hadoop.hive.metastore.api.Table table = getTempTable(tbl.getDbName(), tbl.getTableName());
+ return table != null ? deepCopy(table) : tbl;
+ }
+ return super.getTranslateTableDryrun(tbl);
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 37386c93df1..e68c76f4043 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -7793,7 +7793,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
try {
if (tblDesc != null) {
- destinationTable = db.getTranslateTableDryrun(tblDesc.toTable(conf).getTTable());
+ Table t = tblDesc.toTable(conf);
+ destinationTable = tblDesc.isMaterialization() ? t : db.getTranslateTableDryrun(t.getTTable());
} else {
destinationTable = viewDesc != null ? viewDesc.toTable(conf) : null;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 3b489535c33..1095b13e1a3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -516,7 +516,9 @@ public abstract class TaskCompiler {
}
try {
Table table = ctd.toTable(conf);
- table = db.getTranslateTableDryrun(table.getTTable());
+ if (!ctd.isMaterialization()) {
+ table = db.getTranslateTableDryrun(table.getTTable());
+ }
org.apache.hadoop.hive.metastore.api.Table tTable = table.getTTable();
if (tTable.getSd() != null && tTable.getSd().getLocation() != null) {
location = new Path(tTable.getSd().getLocation());
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestTempAcidTable.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestTempAcidTable.java
new file mode 100755
index 00000000000..5d7ee356c29
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestTempAcidTable.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.metadata;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.util.StringUtils;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+
+import java.util.Map;
+
+/**
+ * TestTempAcidTable.
+ *
+ */
+public class TestTempAcidTable {
+ private static Hive hive;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ hive = Hive.get();
+ HiveConf hiveConf = hive.getConf();
+ hiveConf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER,
+ "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
+ hiveConf.setBoolVar(ConfVars.HIVE_IN_TEST, true);
+ hiveConf.setVar(ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
+ hiveConf.setVar(ConfVars.METASTORE_CLIENT_CAPABILITIES, "HIVEFULLACIDWRITE,HIVEMANAGEDINSERTWRITE");
+ MetastoreConf.setVar(hiveConf, MetastoreConf.ConfVars.METASTORE_METADATA_TRANSFORMER_CLASS,
+ "org.apache.hadoop.hive.metastore.MetastoreDefaultTransformer");
+ SessionState ss = SessionState.start(hiveConf);
+ ss.initTxnMgr(hive.getConf());
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ Hive.closeCurrent();
+ }
+
+ @Test
+ public void testTempInsertOnlyTableTranslate() throws Throwable {
+ try {
+ String dbName = Warehouse.DEFAULT_DATABASE_NAME;
+ String tableName = "temp_table";
+ hive.dropTable(dbName, tableName);
+ Table table = new Table(dbName, tableName);
+ table.setInputFormatClass(OrcInputFormat.class);
+ table.setOutputFormatClass(OrcOutputFormat.class);
+ Map<String, String> params = table.getParameters();
+ params.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
+ params.put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, "insert_only");
+ table.setTemporary(true);
+ hive.createTable(table);
+
+ Table table1 = hive.getTable(dbName, tableName);
+ assertNotNull(table1);
+ assertEquals(tableName, table1.getTableName());
+ assertTrue(table.isTemporary());
+
+ Table table2 = hive.getTranslateTableDryrun(table1.getTTable());
+ assertNotNull(table2);
+ assertEquals(table1.getTTable().getSd().getLocation(), table2.getTTable().getSd().getLocation());
+
+ hive.dropTable(dbName, tableName);
+ } catch (Throwable e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testTempInsertOnlyAcidTableTranslate() failed");
+ throw e;
+ }
+ }
+
+ @Test
+ public void testTempFullAcidTableTranslate() throws Throwable {
+ try {
+ String dbName = Warehouse.DEFAULT_DATABASE_NAME;
+ String tableName = "temp_table";
+ hive.dropTable(dbName, tableName);
+ Table table = new Table(dbName, tableName);
+ table.setInputFormatClass(OrcInputFormat.class);
+ table.setOutputFormatClass(OrcOutputFormat.class);
+ Map<String, String> params = table.getParameters();
+ params.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true");
+ table.setTemporary(true);
+ hive.createTable(table);
+
+ Table table1 = hive.getTable(dbName, tableName);
+ assertNotNull(table1);
+ assertEquals(tableName, table1.getTableName());
+ assertTrue(table.isTemporary());
+
+ Table table2 = hive.getTranslateTableDryrun(table1.getTTable());
+ assertNotNull(table2);
+ assertEquals(table1.getTTable().getSd().getLocation(), table2.getTTable().getSd().getLocation());
+
+ hive.dropTable(dbName, tableName);
+ } catch (Throwable e) {
+ System.err.println(StringUtils.stringifyException(e));
+ System.err.println("testTempFullAcidTableTranslate() failed");
+ throw e;
+ }
+ }
+}