You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ma...@apache.org on 2019/03/22 06:53:58 UTC
[hive] branch master updated: HIVE-21430 : INSERT into a
dynamically partitioned table with hive.stats.autogather = false throws a
MetaException. (Ashutosh Bapat, reviewed by Mahesh Kumar Behera )
This is an automated email from the ASF dual-hosted git repository.
mahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4c87512 HIVE-21430 : INSERT into a dynamically partitioned table with hive.stats.autogather = false throws a MetaException. (Ashutosh Bapat, reviewed by Mahesh Kumar Behera )
4c87512 is described below
commit 4c875127f6f07761af4e1ee146f7bc13738b6276
Author: Ashutosh Bapat <ab...@cloudera.com>
AuthorDate: Fri Mar 22 12:15:52 2019 +0530
HIVE-21430 : INSERT into a dynamically partitioned table with hive.stats.autogather = false throws a MetaException. (Ashutosh Bapat, reviewed by Mahesh Kumar Behera )
Signed-off-by: Mahesh Kumar Behera <ma...@apache.org>
---
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 42 +++++++++++++---------
.../hive/ql/stats/TestStatsUpdaterThread.java | 25 +++++++++++++
2 files changed, 51 insertions(+), 16 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 33d157d..4350dc8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1882,6 +1882,20 @@ public class Hive {
return getDatabase(currentDb);
}
+ private TableSnapshot getTableSnapshot(Table tbl, Long writeId) throws LockException {
+ TableSnapshot tableSnapshot = null;
+ if ((writeId != null) && (writeId > 0)) {
+ ValidWriteIdList writeIds = AcidUtils.getTableValidWriteIdListWithTxnList(
+ conf, tbl.getDbName(), tbl.getTableName());
+ tableSnapshot = new TableSnapshot(writeId, writeIds.writeToString());
+ } else {
+ // Make sure we pass in the names, so we can get the correct snapshot for rename table.
+ tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, tbl.getDbName(), tbl.getTableName(),
+ true);
+ }
+ return tableSnapshot;
+ }
+
/**
* Load a directory into a Hive Table Partition - Alters existing content of
* the partition with the contents of loadPath. - If the partition does not
@@ -1935,17 +1949,7 @@ public class Hive {
inheritLocation, isSkewedStoreAsSubdir, isSrcLocal, isAcidIUDoperation,
resetStatistics, writeId, stmtId, isInsertOverwrite, isTxnTable, newFiles);
- AcidUtils.TableSnapshot tableSnapshot = null;
- if (isTxnTable) {
- if ((writeId != null) && (writeId > 0)) {
- ValidWriteIdList writeIds = AcidUtils.getTableValidWriteIdListWithTxnList(
- conf, tbl.getDbName(), tbl.getTableName());
- tableSnapshot = new TableSnapshot(writeId, writeIds.writeToString());
- } else {
- // Make sure we pass in the names, so we can get the correct snapshot for rename table.
- tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, tbl.getDbName(), tbl.getTableName(), true);
- }
- }
+ AcidUtils.TableSnapshot tableSnapshot = isTxnTable ? getTableSnapshot(tbl, writeId) : null;
if (tableSnapshot != null) {
newTPart.getTPartition().setWriteId(tableSnapshot.getWriteId());
}
@@ -2363,7 +2367,8 @@ public class Hive {
private void setStatsPropAndAlterPartitions(boolean resetStatistics, Table tbl,
List<Partition> partitions,
- long writeId) throws TException {
+ AcidUtils.TableSnapshot tableSnapshot)
+ throws TException {
if (partitions.isEmpty()) {
return;
}
@@ -2377,9 +2382,15 @@ public class Hive {
LOG.debug(sb.toString());
}
+ String validWriteIdList = null;
+ long writeId = 0L;
+ if (tableSnapshot != null) {
+ validWriteIdList = tableSnapshot.getValidWriteIdList();
+ writeId = tableSnapshot.getWriteId();
+ }
getSynchronizedMSC().alter_partitions(tbl.getCatName(), tbl.getDbName(), tbl.getTableName(),
partitions.stream().map(Partition::getTPartition).collect(Collectors.toList()),
- ec, null, writeId);
+ ec, validWriteIdList, writeId);
}
/**
@@ -2626,6 +2637,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
boolean isTxnTable = AcidUtils.isTransactionalTable(tbl);
+ AcidUtils.TableSnapshot tableSnapshot = isTxnTable ? getTableSnapshot(tbl, writeId) : null;
for (Entry<Path, PartitionDetails> entry : partitionDetailsMap.entrySet()) {
tasks.add(() -> {
@@ -2645,8 +2657,6 @@ private void constructOneLBLocationMap(FileStatus fSta,
// if the partition already existed before the loading, no need to add it again to the
// metastore
- AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf,
- partition.getTable(), true);
if (tableSnapshot != null) {
partition.getTPartition().setWriteId(tableSnapshot.getWriteId());
}
@@ -2722,7 +2732,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
partitionDetailsMap.entrySet().stream()
.filter(entry -> entry.getValue().hasOldPartition)
.map(entry -> entry.getValue().partition)
- .collect(Collectors.toList()), writeId);
+ .collect(Collectors.toList()), tableSnapshot);
} catch (InterruptedException | ExecutionException e) {
throw new HiveException("Exception when loading " + validPartitions.size()
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
index 24acd6d..a2f8bab 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
@@ -290,6 +290,31 @@ public class TestStatsUpdaterThread {
msClient.close();
}
+ @Test
+ public void testTxnDynamicPartitions() throws Exception {
+ StatsUpdaterThread su = createUpdater();
+ IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf);
+
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false);
+ executeQuery("create table simple_stats (s string) partitioned by (i int)" +
+ " stored as orc " +
+ " TBLPROPERTIES (\"transactional\"=\"true\")");
+ executeQuery("insert into simple_stats (i, s) values (1, 'test')");
+ executeQuery("insert into simple_stats (i, s) values (2, 'test2')");
+ executeQuery("insert into simple_stats (i, s) values (3, 'test3')");
+ assertTrue(su.runOneIteration());
+ drainWorkQueue(su);
+ verifyPartStatsUpToDate(3, 1, msClient, "simple_stats", true);
+
+ executeQuery("insert into simple_stats (i, s) values (1, 'test12')");
+ executeQuery("insert into simple_stats (i, s) values (2, 'test22')");
+ executeQuery("insert into simple_stats (i, s) values (3, 'test32')");
+ assertTrue(su.runOneIteration());
+ drainWorkQueue(su);
+ verifyPartStatsUpToDate(3, 1, msClient, "simple_stats", true);
+ msClient.close();
+ }
+
@Test(timeout=40000)
public void testExistingOnly() throws Exception {
hiveConf.set(MetastoreConf.ConfVars.STATS_AUTO_UPDATE.getVarname(), "existing");