You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2022/12/19 12:32:45 UTC
[hive] branch master updated: HIVE-26788: Update stats of table/partition using noscan operation when stats autogather is enabled (Sourabh Badhya, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 07a47563d6c HIVE-26788: Update stats of table/partition using noscan operation when stats autogather is enabled (Sourabh Badhya, reviewed by Denys Kuzmenko)
07a47563d6c is described below
commit 07a47563d6cd8a0f01e04b2b0d39693ec720013b
Author: Sourabh Badhya <42...@users.noreply.github.com>
AuthorDate: Mon Dec 19 18:02:32 2022 +0530
HIVE-26788: Update stats of table/partition using noscan operation when stats autogather is enabled (Sourabh Badhya, reviewed by Denys Kuzmenko)
Closes #3812
---
.../org/apache/hadoop/hive/ql/TestAcidOnTez.java | 2 +-
.../hive/ql/txn/compactor/TestCompactor.java | 18 ++--
.../ql/txn/compactor/TestCrudCompactorOnTez.java | 104 +++++++++++++++++++--
.../ql/txn/compactor/TestMmCompactorOnTez.java | 4 +-
.../hadoop/hive/ql/txn/compactor/StatsUpdater.java | 18 +++-
.../hadoop/hive/ql/txn/compactor/Worker.java | 4 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 6 +-
.../apache/hadoop/hive/ql/TestTxnCommands3.java | 4 +-
.../org/apache/hadoop/hive/ql/TestTxnLoadData.java | 22 ++---
.../hadoop/hive/ql/txn/compactor/TestWorker.java | 2 +-
10 files changed, 142 insertions(+), 42 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
index 5671d4a6daa..442de012473 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
@@ -570,7 +570,7 @@ public class TestAcidOnTez {
for(int i = 0; i < expected2.length; i++) {
Assert.assertTrue("Actual line " + i + " bc: " + rs.get(i), rs.get(i).startsWith(expected2[i][0]));
//everything is now in base/
- Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith("base_0000003_v0000025/bucket_00000"));
+ Assert.assertTrue("Actual line(file) " + i + " bc: " + rs.get(i), rs.get(i).endsWith("base_0000003_v0000026/bucket_00000"));
}
}
/**
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index e1a72497c41..f7cd620ec32 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -335,6 +335,7 @@ public class TestCompactor extends TestCompactorBase {
//as of (8/27/2014) Hive 0.14, ACID/Orc requires HiveInputFormat
String dbName = "default";
String tblName = "compaction_test";
+ IMetaStoreClient msClient = new HiveMetaStoreClient(conf);
executeStatementOnDriver("drop table if exists " + tblName, driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " +
" PARTITIONED BY(bkt INT)" +
@@ -372,11 +373,11 @@ public class TestCompactor extends TestCompactorBase {
//compute stats before compaction
CompactionInfo ci = new CompactionInfo(dbName, tblName, "bkt=0", CompactionType.MAJOR);
- statsUpdater.gatherStats(ci, conf,
- System.getProperty("user.name"), CompactorUtil.getCompactorJobQueueName(conf, ci, table));
+ statsUpdater.gatherStats(ci, conf, System.getProperty("user.name"),
+ CompactorUtil.getCompactorJobQueueName(conf, ci, table), msClient);
ci = new CompactionInfo(dbName, tblName, "bkt=1", CompactionType.MAJOR);
- statsUpdater.gatherStats(ci, conf,
- System.getProperty("user.name"), CompactorUtil.getCompactorJobQueueName(conf, ci, table));
+ statsUpdater.gatherStats(ci, conf, System.getProperty("user.name"),
+ CompactorUtil.getCompactorJobQueueName(conf, ci, table), msClient);
//Check basic stats are collected
org.apache.hadoop.hive.ql.metadata.Table hiveTable = Hive.get().getTable(tblName);
@@ -453,6 +454,7 @@ public class TestCompactor extends TestCompactorBase {
//as of (8/27/2014) Hive 0.14, ACID/Orc requires HiveInputFormat
String dbName = "default";
String tblName = "compaction_test";
+ IMetaStoreClient msClient = new HiveMetaStoreClient(conf);
executeStatementOnDriver("drop table if exists " + tblName, driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " +
" CLUSTERED BY(a) INTO 4 BUCKETS" + //currently ACID requires table to be bucketed
@@ -469,8 +471,8 @@ public class TestCompactor extends TestCompactorBase {
//compute stats before compaction
CompactionInfo ci = new CompactionInfo(dbName, tblName, null, CompactionType.MAJOR);
- statsUpdater.gatherStats(ci, conf,
- System.getProperty("user.name"), CompactorUtil.getCompactorJobQueueName(conf, ci, table));
+ statsUpdater.gatherStats(ci, conf, System.getProperty("user.name"),
+ CompactorUtil.getCompactorJobQueueName(conf, ci, table), msClient);
//Check basic stats are collected
Map<String, String> parameters = Hive.get().getTable(tblName).getParameters();
@@ -2098,10 +2100,10 @@ public class TestCompactor extends TestCompactorBase {
txnHandler.compact(new CompactionRequest(dbName, tableName, CompactionType.MAJOR));
runWorker(conf);
- // Make sure the statistics is NOT updated for the table (compaction triggers only a basic stats gathering)
+ // Make sure the statistics is updated for the table
colStats = msClient.getTableColumnStatistics(dbName, tableName, colNames, Constants.HIVE_ENGINE);
assertEquals("Stats should be there", 1, colStats.size());
- assertEquals("Value should contain new data", 1, colStats.get(0).getStatsData().getLongStats().getHighValue());
+ assertEquals("Value should contain new data", 2, colStats.get(0).getStatsData().getLongStats().getHighValue());
assertEquals("Value should contain new data", 1, colStats.get(0).getStatsData().getLongStats().getLowValue());
}
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
index f63a6a8f922..00ddb5c47a2 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
@@ -320,6 +320,7 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest {
//as of (8/27/2014) Hive 0.14, ACID/Orc requires HiveInputFormat
String dbName = "default";
String tblName = "compaction_test";
+ IMetaStoreClient msClient = new HiveMetaStoreClient(conf);
executeStatementOnDriver("drop table if exists " + tblName, driver);
executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " +
" CLUSTERED BY(a) INTO 4 BUCKETS" + //currently ACID requires table to be bucketed
@@ -336,8 +337,8 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest {
//compute stats before compaction
CompactionInfo ci = new CompactionInfo(dbName, tblName, null, CompactionType.MAJOR);
- new StatsUpdater().gatherStats(ci, conf,
- System.getProperty("user.name"), CompactorUtil.getCompactorJobQueueName(conf, ci, table));
+ new StatsUpdater().gatherStats(ci, conf, System.getProperty("user.name"),
+ CompactorUtil.getCompactorJobQueueName(conf, ci, table), msClient);
//Check basic stats are collected
Map<String, String> parameters = Hive.get().getTable(tblName).getParameters();
@@ -1044,7 +1045,7 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest {
List<String> actualBasesAfterComp =
CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.baseFileFilter, table, null);
Assert.assertEquals("Base directory does not match after compaction",
- Collections.singletonList("base_0000008_v0000038"), actualBasesAfterComp);
+ Collections.singletonList("base_0000008_v0000039"), actualBasesAfterComp);
// Verify bucket files in delta dirs
Assert.assertEquals("Bucket names are not matching after compaction", expectedBucketFiles,
CompactorTestUtil.getBucketFileNames(fs, table, null, actualBasesAfterComp.get(0)));
@@ -1396,11 +1397,11 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest {
List<String> actualDeltasAfterComp =
CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deltaFileFilter, table, null);
Assert.assertEquals("Delta directories does not match after compaction",
- Collections.singletonList("delta_0000001_0000015_v0000044"), actualDeltasAfterComp);
+ Collections.singletonList("delta_0000001_0000015_v0000046"), actualDeltasAfterComp);
List<String> actualDeleteDeltasAfterComp =
CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deleteEventDeltaDirFilter, table, null);
Assert.assertEquals("Delete delta directories does not match after compaction",
- Collections.singletonList("delete_delta_0000001_0000015_v0000044"), actualDeleteDeltasAfterComp);
+ Collections.singletonList("delete_delta_0000001_0000015_v0000046"), actualDeleteDeltasAfterComp);
CompactorTestUtilities.checkAcidVersion(fs.listFiles(new Path(table.getSd().getLocation()), true), fs,
conf.getBoolVar(HiveConf.ConfVars.HIVE_WRITE_ACID_VERSION_FILE),
@@ -1581,7 +1582,7 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest {
verifySuccessfulCompaction(2);
// Verify base directory after compaction
Assert.assertEquals("Base directory does not match after major compaction",
- Collections.singletonList("base_0000010_v0000029"),
+ Collections.singletonList("base_0000010_v0000030"),
CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.baseFileFilter, table, null));
// Verify all contents
actualData = dataProvider.getAllData(tableName);
@@ -1931,7 +1932,7 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest {
// 2 compactions should be in the response queue with succeeded state
verifySuccessfulCompaction(2);
// Should contain only one base directory now
- String expectedBase = "base_0000006_v0000023";
+ String expectedBase = "base_0000006_v0000024";
Assert.assertEquals("Base directory does not match after major compaction",
Collections.singletonList(expectedBase),
CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.baseFileFilter, table, null));
@@ -2056,12 +2057,12 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest {
List<String> actualDeltasAfterComp =
CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deltaFileFilter, table, null);
Assert.assertEquals("Delta directory does not match after compaction",
- Collections.singletonList("delta_0000001_0000004_v0000032"), actualDeltasAfterComp);
+ Collections.singletonList("delta_0000001_0000004_v0000033"), actualDeltasAfterComp);
List<String> actualDeleteDeltasAfterComp =
CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deleteEventDeltaDirFilter, table, null);
Assert.assertEquals("Delete delta directory does not match after compaction",
- Collections.singletonList("delete_delta_0000001_0000004_v0000032"), actualDeleteDeltasAfterComp);
+ Collections.singletonList("delete_delta_0000001_0000004_v0000033"), actualDeleteDeltasAfterComp);
// Verify bucket files in delta dirs
List<String> expectedBucketFiles = Collections.singletonList("bucket_00000");
@@ -2609,4 +2610,89 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest {
List<String> actualData = testDP.getAllData(tblName);
Assert.assertEquals(expectedData, actualData);
}
+
+ @Test
+ public void testStatsAfterCompactionPartTblForMRCompaction() throws Exception {
+ testStatsAfterCompactionPartTbl(false, true, CompactionType.MINOR);
+ testStatsAfterCompactionPartTbl(false, false, CompactionType.MAJOR);
+ }
+
+ @Test
+ public void testStatsAfterCompactionPartTblForQueryBasedCompaction() throws Exception {
+ testStatsAfterCompactionPartTbl(true, true, CompactionType.MINOR);
+ testStatsAfterCompactionPartTbl(true, false, CompactionType.MAJOR);
+ }
+
+ public void testStatsAfterCompactionPartTbl(boolean isQueryBased, boolean isAutoGatherStats,
+ CompactionType compactionType) throws Exception {
+ conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, isQueryBased);
+ conf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, isAutoGatherStats);
+ String dbName = "default";
+ String tblName = "minor_compaction_test";
+ IMetaStoreClient msClient = new HiveMetaStoreClient(conf);
+ executeStatementOnDriver("drop table if exists " + tblName, driver);
+ executeStatementOnDriver("CREATE TABLE " + tblName + "(a INT, b STRING) " +
+ " PARTITIONED BY(bkt INT)" +
+ " STORED AS ORC TBLPROPERTIES ('transactional'='true')", driver);
+
+ executeStatementOnDriver("INSERT INTO TABLE " + tblName + " PARTITION(bkt=1)" +
+ " values(57, 'Budapest')", driver);
+ executeStatementOnDriver("INSERT INTO TABLE " + tblName + " PARTITION(bkt=1)" +
+ " values(58, 'Milano')", driver);
+ executeStatementOnDriver("INSERT INTO TABLE " + tblName + " PARTITION(bkt=1)" +
+ " values(59, 'Bangalore')", driver);
+ executeStatementOnDriver("INSERT INTO TABLE " + tblName + " PARTITION(bkt=1)" +
+ " values(60, 'New York')", driver);
+ executeStatementOnDriver("DELETE FROM " + tblName + " WHERE a = 57", driver);
+ executeStatementOnDriver("DELETE FROM " + tblName + " WHERE a = 58", driver);
+
+ TxnStore txnHandler = TxnUtils.getTxnStore(conf);
+ Table table = msClient.getTable(dbName, tblName);
+
+ //compute stats before compaction
+ CompactionInfo ci = new CompactionInfo(dbName, tblName, "bkt=1", compactionType);
+ new StatsUpdater().gatherStats(ci, conf, System.getProperty("user.name"),
+ CompactorUtil.getCompactorJobQueueName(conf, ci, table), msClient);
+
+ //Check basic stats are collected
+ org.apache.hadoop.hive.ql.metadata.Table hiveTable = Hive.get().getTable(tblName);
+ List<org.apache.hadoop.hive.ql.metadata.Partition> partitions = Hive.get().getPartitions(hiveTable);
+ Map<String, String> parameters = partitions
+ .stream()
+ .filter(p -> p.getName().equals("bkt=1"))
+ .findFirst()
+ .orElseThrow(() -> new RuntimeException("Could not get Partition"))
+ .getParameters();
+ Assert.assertEquals("The number of files is differing from the expected", "6", parameters.get("numFiles"));
+ Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows"));
+
+ //Do a minor compaction
+ CompactionRequest rqst = new CompactionRequest(dbName, tblName, compactionType);
+ rqst.setPartitionname("bkt=1");
+ txnHandler.compact(rqst);
+ runWorker(conf);
+ CompactorTestUtil.runCleaner(conf);
+
+ ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+ List<ShowCompactResponseElement> compacts = rsp.getCompacts();
+ if (1 != compacts.size()) {
+ Assert.fail("Expecting 1 compaction and found " + compacts.size() + " compactions " + compacts);
+ }
+ Assert.assertEquals("succeeded", compacts.get(0).getState());
+
+ partitions = Hive.get().getPartitions(hiveTable);
+ parameters = partitions
+ .stream()
+ .filter(p -> p.getName().equals("bkt=1"))
+ .findFirst()
+ .orElseThrow(() -> new RuntimeException("Could not get Partition"))
+ .getParameters();
+ if (compactionType == CompactionType.MINOR) {
+ Assert.assertEquals("The number of files is differing from the expected", "2", parameters.get("numFiles"));
+ } else {
+ Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles"));
+ }
+ Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows"));
+ executeStatementOnDriver("drop table if exists " + tblName, driver);
+ }
}
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMmCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMmCompactorOnTez.java
index 35a998248c1..55a57da9785 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMmCompactorOnTez.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMmCompactorOnTez.java
@@ -350,7 +350,7 @@ public class TestMmCompactorOnTez extends CompactorOnTezTest {
List<String> actualDeltasAfterComp =
CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deltaFileFilter, table, null);
Assert.assertEquals("Delta directories does not match after compaction",
- Collections.singletonList("delta_0000001_0000009_v0000026"), actualDeltasAfterComp);
+ Collections.singletonList("delta_0000001_0000009_v0000028"), actualDeltasAfterComp);
}
@@ -388,7 +388,7 @@ public class TestMmCompactorOnTez extends CompactorOnTezTest {
verifySuccessulTxn(2);
// Verify base directory after compaction
Assert.assertEquals("Base directory does not match after major compaction",
- Collections.singletonList("base_0000006_v0000019"),
+ Collections.singletonList("base_0000006_v0000020"),
CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.baseFileFilter, table, null));
actualData = dataProvider.getAllData(tableName);
Assert.assertEquals(expectedData, actualData);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
index ed2659ab848..698346b9b72 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.txn.compactor;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
import org.apache.hadoop.hive.ql.DriverUtils;
@@ -28,6 +29,7 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.List;
import java.util.Map;
/**
@@ -50,10 +52,12 @@ public final class StatsUpdater {
* @param userName The user to run the statistic collection with
* @param compactionQueueName The name of the compaction queue
*/
- public void gatherStats(CompactionInfo ci, HiveConf conf, String userName, String compactionQueueName) {
+ public void gatherStats(CompactionInfo ci, HiveConf conf,
+ String userName, String compactionQueueName,
+ IMetaStoreClient msc) {
try {
- if (!ci.isMajorCompaction()) {
- return;
+ if (msc == null) {
+ throw new IllegalArgumentException("Metastore client is missing");
}
HiveConf statusUpdaterConf = new HiveConf(conf);
@@ -73,6 +77,14 @@ public final class StatsUpdater {
sb.append(")");
}
sb.append(" compute statistics");
+ if (!conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER) && ci.isMajorCompaction()) {
+ List<String> columnList = msc.findColumnsWithStats(CompactionInfo.compactionInfoToStruct(ci));
+ if (!columnList.isEmpty()) {
+ sb.append(" for columns ").append(String.join(",", columnList));
+ }
+ } else {
+ sb.append(" noscan");
+ }
LOG.info(ci + ": running '" + sb + "'");
if (compactionQueueName != null && compactionQueueName.length() > 0) {
statusUpdaterConf.set(TezConfiguration.TEZ_QUEUE_NAME, compactionQueueName);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index 7667cfc971e..bdea1f455d1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -472,8 +472,8 @@ public class Worker extends RemoteCompactorThread implements MetaStoreThread {
}
if (computeStats) {
- statsUpdater.gatherStats(ci, conf, runJobAsSelf(ci.runAs) ? ci.runAs : t1.getOwner(),
- CompactorUtil.getCompactorJobQueueName(conf, ci, t1));
+ statsUpdater.gatherStats(ci, conf, runJobAsSelf(ci.runAs) ? ci.runAs : t1.getOwner(),
+ CompactorUtil.getCompactorJobQueueName(conf, ci, t1), msc);
}
return true;
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index ccf6bc1df14..b49347bba87 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -1573,8 +1573,8 @@ public class TestTxnCommands2 extends TxnCommandsBaseForTests {
status = fs.listStatus(new Path(getWarehouseDir() + "/" + tblName.toLowerCase()),
FileUtils.HIDDEN_FILES_PATH_FILTER);
expectedDeltas = new HashSet<>();
- expectedDeltas.add("delete_delta_0000001_0000004_v0000025");
- expectedDeltas.add("delta_0000001_0000004_v0000025");
+ expectedDeltas.add("delete_delta_0000001_0000004_v0000026");
+ expectedDeltas.add("delta_0000001_0000004_v0000026");
actualDeltas = new HashSet<>();
for(FileStatus file : status) {
actualDeltas.add(file.getPath().getName());
@@ -3334,7 +3334,7 @@ public class TestTxnCommands2 extends TxnCommandsBaseForTests {
String oldDelta4 = "delta_0000004_0000004_0000";
String expectedDelta1 = p1 + "/delta_0000001_0000002_v0000021";
- String expectedDelta2 = p2 + "/delta_0000003_0000004_v0000022";
+ String expectedDelta2 = p2 + "/delta_0000003_0000004_v0000023";
runStatementOnDriver("insert into " + Table.ACIDTBLPART + " partition(p='p1') (a,b) values(1,2)");
runStatementOnDriver("insert into " + Table.ACIDTBLPART + " partition(p='p1') (a,b) values(3,4)");
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
index 9537f1c083c..91ffa2c4cc2 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
@@ -385,14 +385,14 @@ public class TestTxnCommands3 extends TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'minor'");
runWorker(hiveConf);
/*
- at this point delta_0000001_0000003_v0000022 is visible to everyone
+ at this point delta_0000001_0000003_v0000023 is visible to everyone
so cleaner removes all files shadowed by it (which is everything in this case)
*/
runCleaner(hiveConf);
runCleaner(hiveConf);
expectedList = new String[] {
- "/t/delta_0000001_0000003_v0000022"
+ "/t/delta_0000001_0000003_v0000023"
};
actualList = fs.listStatus(new Path(warehousePath + "/t"),
FileUtils.HIDDEN_FILES_PATH_FILTER);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
index a4d4aeff50e..43f5417839a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnLoadData.java
@@ -173,9 +173,9 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'major'");
TestTxnCommands2.runWorker(hiveConf);
String[][] expected6 = new String[][]{
- {"{\"writeid\":7,\"bucketid\":536870913,\"rowid\":0}\t1\t17", "t/base_0000009_v0000048/bucket_00000"},
- {"{\"writeid\":7,\"bucketid\":536936449,\"rowid\":0}\t1\t17", "t/base_0000009_v0000048/bucket_00001"},
- {"{\"writeid\":9,\"bucketid\":536870912,\"rowid\":0}\t2\t2", "t/base_0000009_v0000048/bucket_00000"}
+ {"{\"writeid\":7,\"bucketid\":536870913,\"rowid\":0}\t1\t17", "t/base_0000009_v0000049/bucket_00000"},
+ {"{\"writeid\":7,\"bucketid\":536936449,\"rowid\":0}\t1\t17", "t/base_0000009_v0000049/bucket_00001"},
+ {"{\"writeid\":9,\"bucketid\":536870912,\"rowid\":0}\t2\t2", "t/base_0000009_v0000049/bucket_00000"}
};
checkResult(expected6, testQuery, isVectorized, "load data inpath compact major");
}
@@ -222,11 +222,11 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'major'");
TestTxnCommands2.runWorker(hiveConf);
String[][] expected2 = new String[][] {
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2", "t/base_0000003_v0000030/bucket_00000"},
- {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4", "t/base_0000003_v0000030/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "t/base_0000003_v0000030/bucket_00000"},
- {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "t/base_0000003_v0000030/bucket_00000"},
- {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2", "t/base_0000003_v0000030/bucket_00000"}
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t0\t2", "t/base_0000003_v0000031/bucket_00000"},
+ {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":1}\t0\t4", "t/base_0000003_v0000031/bucket_00000"},
+ {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":0}\t1\t2", "t/base_0000003_v0000031/bucket_00000"},
+ {"{\"writeid\":2,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "t/base_0000003_v0000031/bucket_00000"},
+ {"{\"writeid\":3,\"bucketid\":536870912,\"rowid\":0}\t2\t2", "t/base_0000003_v0000031/bucket_00000"}
};
checkResult(expected2, testQuery, isVectorized, "load data inpath (major)");
@@ -244,9 +244,9 @@ public class TestTxnLoadData extends TxnCommandsBaseForTests {
runStatementOnDriver("alter table T compact 'major'");
TestTxnCommands2.runWorker(hiveConf);
String[][] expected4 = new String[][] {
- {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":0}\t5\t6", "t/base_0000005_v0000040/bucket_00000"},
- {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":1}\t7\t8", "t/base_0000005_v0000040/bucket_00000"},
- {"{\"writeid\":5,\"bucketid\":536870912,\"rowid\":0}\t6\t6", "t/base_0000005_v0000040/bucket_00000"}};
+ {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":0}\t5\t6", "t/base_0000005_v0000041/bucket_00000"},
+ {"{\"writeid\":4,\"bucketid\":536870912,\"rowid\":1}\t7\t8", "t/base_0000005_v0000041/bucket_00000"},
+ {"{\"writeid\":5,\"bucketid\":536870912,\"rowid\":0}\t6\t6", "t/base_0000005_v0000041/bucket_00000"}};
checkResult(expected4, testQuery, isVectorized, "load data inpath overwrite (major)");
}
/**
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java
index 1b964648029..2574b66e342 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java
@@ -985,7 +985,7 @@ public class TestWorker extends CompactorTest {
worker.findNextCompactionAndExecute(true, true);
- Mockito.verify(statsUpdater, Mockito.never()).gatherStats(any(), any(), any(), any());
+ Mockito.verify(statsUpdater, Mockito.never()).gatherStats(any(), any(), any(), any(), any());
}
@Test