You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2017/05/17 07:21:43 UTC
hive git commit: HIVE-16678 : Truncate on temporary table fails with
"table not found" error. (Sankar Hariappan via Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master 5a0b42537 -> e97734b1c
HIVE-16678 : Truncate on temporary table fails with "table not found" error. (Sankar Hariappan via Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e97734b1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e97734b1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e97734b1
Branch: refs/heads/master
Commit: e97734b1ca010d9690bbb0d6bdddf92082a20688
Parents: 5a0b425
Author: Sankar Hariappan <ma...@gmail.com>
Authored: Wed May 17 00:21:39 2017 -0700
Committer: Thejas M Nair <th...@hortonworks.com>
Committed: Wed May 17 00:21:39 2017 -0700
----------------------------------------------------------------------
.../ql/metadata/SessionHiveMetaStoreClient.java | 76 ++++++++++
.../clientpositive/temp_table_truncate.q | 21 +++
.../clientpositive/temp_table_truncate.q.out | 143 +++++++++++++++++++
3 files changed, 240 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e97734b1/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index a319b88..4642ec2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -30,8 +30,13 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.io.HdfsUtils;
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
@@ -56,6 +61,8 @@ import org.apache.hadoop.hive.metastore.api.TableMeta;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
import org.apache.thrift.TException;
@@ -115,6 +122,18 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
}
@Override
+ public void truncateTable(String dbName, String tableName, List<String> partNames) throws MetaException, TException {
+ // First try temp table
+ org.apache.hadoop.hive.metastore.api.Table table = getTempTable(dbName, tableName);
+ if (table != null) {
+ truncateTempTable(table);
+ return;
+ }
+ // Try underlying client
+ super.truncateTable(dbName, tableName, partNames);
+ }
+
+ @Override
public org.apache.hadoop.hive.metastore.api.Table getTable(String dbname, String name) throws MetaException,
TException, NoSuchObjectException {
// First check temp tables
@@ -509,6 +528,63 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
return false;
}
+ private boolean needToUpdateStats(Map<String,String> props, EnvironmentContext environmentContext) {
+ if (null == props) {
+ return false;
+ }
+ boolean statsPresent = false;
+ for (String stat : StatsSetupConst.supportedStats) {
+ String statVal = props.get(stat);
+ if (statVal != null && Long.parseLong(statVal) > 0) {
+ statsPresent = true;
+ //In the case of truncate table, we set the stats to be 0.
+ props.put(stat, "0");
+ }
+ }
+ //first set basic stats to true
+ StatsSetupConst.setBasicStatsState(props, StatsSetupConst.TRUE);
+ environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK);
+ //then invalidate column stats
+ StatsSetupConst.clearColumnStatsState(props);
+ return statsPresent;
+ }
+
+ private void truncateTempTable(org.apache.hadoop.hive.metastore.api.Table table) throws MetaException, TException {
+
+ boolean isAutopurge = "true".equalsIgnoreCase(table.getParameters().get("auto.purge"));
+ try {
+ // this is not transactional
+ Path location = new Path(table.getSd().getLocation());
+
+ FileSystem fs = location.getFileSystem(conf);
+ HadoopShims.HdfsEncryptionShim shim
+ = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf);
+ if (!shim.isPathEncrypted(location)) {
+ HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location);
+ FileStatus targetStatus = fs.getFileStatus(location);
+ String targetGroup = targetStatus == null ? null : targetStatus.getGroup();
+ FileUtils.moveToTrash(fs, location, conf, isAutopurge);
+ fs.mkdirs(location);
+ HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false);
+ } else {
+ FileStatus[] statuses = fs.listStatus(location, FileUtils.HIDDEN_FILES_PATH_FILTER);
+ if ((statuses != null) && (statuses.length > 0)) {
+ boolean success = Hive.trashFiles(fs, statuses, conf, isAutopurge);
+ if (!success) {
+ throw new HiveException("Error in deleting the contents of " + location.toString());
+ }
+ }
+ }
+
+ EnvironmentContext environmentContext = new EnvironmentContext();
+ if (needToUpdateStats(table.getParameters(), environmentContext)) {
+ alter_table_with_environmentContext(table.getDbName(), table.getTableName(), table, environmentContext);
+ }
+ } catch (Exception e) {
+ throw new MetaException(e.getMessage());
+ }
+ }
+
private void dropTempTable(org.apache.hadoop.hive.metastore.api.Table table, boolean deleteData,
EnvironmentContext envContext) throws MetaException, TException,
NoSuchObjectException, UnsupportedOperationException {
http://git-wip-us.apache.org/repos/asf/hive/blob/e97734b1/ql/src/test/queries/clientpositive/temp_table_truncate.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/temp_table_truncate.q b/ql/src/test/queries/clientpositive/temp_table_truncate.q
new file mode 100644
index 0000000..7b0fefc
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/temp_table_truncate.q
@@ -0,0 +1,21 @@
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+CREATE TEMPORARY TABLE tmp_src AS SELECT * FROM src WHERE key % 2 = 0;
+CREATE TEMPORARY TABLE tmp_srcpart AS SELECT * FROM srcpart;
+
+DESCRIBE tmp_src;
+DESCRIBE tmp_srcpart;
+SHOW TABLES;
+
+SELECT count(*) FROM tmp_src;
+SELECT count(*) FROM tmp_srcpart;
+
+EXPLAIN TRUNCATE TABLE tmp_src;
+TRUNCATE TABLE tmp_src;
+
+SELECT count(*) FROM tmp_src;
+
+EXPLAIN TRUNCATE TABLE tmp_srcpart;
+TRUNCATE TABLE tmp_srcpart;
+
+SELECT count(*) FROM tmp_srcpart;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/e97734b1/ql/src/test/results/clientpositive/temp_table_truncate.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/temp_table_truncate.q.out b/ql/src/test/results/clientpositive/temp_table_truncate.q.out
new file mode 100644
index 0000000..b1af432
--- /dev/null
+++ b/ql/src/test/results/clientpositive/temp_table_truncate.q.out
@@ -0,0 +1,143 @@
+PREHOOK: query: CREATE TEMPORARY TABLE tmp_src AS SELECT * FROM src WHERE key % 2 = 0
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tmp_src
+POSTHOOK: query: CREATE TEMPORARY TABLE tmp_src AS SELECT * FROM src WHERE key % 2 = 0
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tmp_src
+PREHOOK: query: CREATE TEMPORARY TABLE tmp_srcpart AS SELECT * FROM srcpart
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tmp_srcpart
+POSTHOOK: query: CREATE TEMPORARY TABLE tmp_srcpart AS SELECT * FROM srcpart
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tmp_srcpart
+PREHOOK: query: DESCRIBE tmp_src
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@tmp_src
+POSTHOOK: query: DESCRIBE tmp_src
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@tmp_src
+key string
+value string
+PREHOOK: query: DESCRIBE tmp_srcpart
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@tmp_srcpart
+POSTHOOK: query: DESCRIBE tmp_srcpart
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@tmp_srcpart
+key string
+value string
+ds string
+hr string
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+alltypesorc
+cbo_t1
+cbo_t2
+cbo_t3
+lineitem
+part
+src
+src1
+src_cbo
+src_json
+src_sequencefile
+src_thrift
+srcbucket
+srcbucket2
+srcpart
+tmp_src
+tmp_srcpart
+PREHOOK: query: SELECT count(*) FROM tmp_src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*) FROM tmp_src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_src
+#### A masked pattern was here ####
+247
+PREHOOK: query: SELECT count(*) FROM tmp_srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_srcpart
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*) FROM tmp_srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_srcpart
+#### A masked pattern was here ####
+2000
+PREHOOK: query: EXPLAIN TRUNCATE TABLE tmp_src
+PREHOOK: type: TRUNCATETABLE
+POSTHOOK: query: EXPLAIN TRUNCATE TABLE tmp_src
+POSTHOOK: type: TRUNCATETABLE
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Truncate Table Operator:
+ Truncate Table or Partition
+ TableName: tmp_src
+
+PREHOOK: query: TRUNCATE TABLE tmp_src
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@tmp_src
+POSTHOOK: query: TRUNCATE TABLE tmp_src
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@tmp_src
+PREHOOK: query: SELECT count(*) FROM tmp_src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*) FROM tmp_src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_src
+#### A masked pattern was here ####
+0
+PREHOOK: query: EXPLAIN TRUNCATE TABLE tmp_srcpart
+PREHOOK: type: TRUNCATETABLE
+POSTHOOK: query: EXPLAIN TRUNCATE TABLE tmp_srcpart
+POSTHOOK: type: TRUNCATETABLE
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Truncate Table Operator:
+ Truncate Table or Partition
+ TableName: tmp_srcpart
+
+PREHOOK: query: TRUNCATE TABLE tmp_srcpart
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@tmp_srcpart
+POSTHOOK: query: TRUNCATE TABLE tmp_srcpart
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@tmp_srcpart
+PREHOOK: query: SELECT count(*) FROM tmp_srcpart
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_srcpart
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT count(*) FROM tmp_srcpart
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_srcpart
+#### A masked pattern was here ####
+0