You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ab...@apache.org on 2022/12/21 07:39:08 UTC
[hive] branch master updated: HIVE-26871: TestCrudCompactorOnTez is flaky after HIVE-26479 (#3876) (Sourabh Badhya reviewed by Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new f20e3590b64 HIVE-26871: TestCrudCompactorOnTez is flaky after HIVE-26479 (#3876) (Sourabh Badhya reviewed by Laszlo Bodor)
f20e3590b64 is described below
commit f20e3590b64d474cd2ed024463ae2aefbb3f3144
Author: Sourabh Badhya <42...@users.noreply.github.com>
AuthorDate: Wed Dec 21 13:08:55 2022 +0530
HIVE-26871: TestCrudCompactorOnTez is flaky after HIVE-26479 (#3876) (Sourabh Badhya reviewed by Laszlo Bodor)
---
.../hive/ql/txn/compactor/CompactorOnTezTest.java | 29 +++++++++++--------
.../ql/txn/compactor/TestMmCompactorOnTez.java | 4 ++-
.../hive/ql/hooks/TestHiveProtoLoggingHook.java | 33 ++++++++++++++--------
3 files changed, 42 insertions(+), 24 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorOnTezTest.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorOnTezTest.java
index 8d378053e71..5b92199f558 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorOnTezTest.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorOnTezTest.java
@@ -171,21 +171,26 @@ public abstract class CompactorOnTezTest {
}
protected HiveHookEvents.HiveHookEventProto getRelatedTezEvent(String dbTableName) throws Exception {
- ProtoMessageReader<HiveHookEvents.HiveHookEventProto> reader = TestHiveProtoLoggingHook.getTestReader(conf, tmpFolder);
- HiveHookEvents.HiveHookEventProto event = reader.readEvent();
- boolean getRelatedEvent = false;
- while (!getRelatedEvent) {
- while (ExecutionMode.TEZ != ExecutionMode.valueOf(event.getExecutionMode())) {
- event = reader.readEvent();
+ List<ProtoMessageReader<HiveHookEvents.HiveHookEventProto>> readers = TestHiveProtoLoggingHook.getTestReader(conf, tmpFolder);
+ for (ProtoMessageReader<HiveHookEvents.HiveHookEventProto> reader : readers) {
+ HiveHookEvents.HiveHookEventProto event = reader.readEvent();
+ boolean getRelatedEvent = false;
+ while (!getRelatedEvent) {
+ while (event != null && ExecutionMode.TEZ != ExecutionMode.valueOf(event.getExecutionMode())) {
+ event = reader.readEvent();
+ }
+ // Tables read is the table picked for compaction.
+ if (event.getTablesReadCount() > 0 && dbTableName.equalsIgnoreCase(event.getTablesRead(0))) {
+ getRelatedEvent = true;
+ } else {
+ event = reader.readEvent();
+ }
}
- // Tables read is the table picked for compaction.
- if (event.getTablesReadCount() > 0 && dbTableName.equalsIgnoreCase(event.getTablesRead(0))) {
- getRelatedEvent = true;
- } else {
- event = reader.readEvent();
+ if (getRelatedEvent) {
+ return event;
}
}
- return event;
+ return null;
}
protected class TestDataProvider {
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMmCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMmCompactorOnTez.java
index 55a57da9785..40b49f2df36 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMmCompactorOnTez.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMmCompactorOnTez.java
@@ -107,7 +107,9 @@ public class TestMmCompactorOnTez extends CompactorOnTezTest {
testDataProvider.dropTable(tableName);
if (isTez(conf)) {
- ProtoMessageReader<HiveHookEvents.HiveHookEventProto> reader = TestHiveProtoLoggingHook.getTestReader(conf, tmpFolder);
+ List<ProtoMessageReader<HiveHookEvents.HiveHookEventProto>> readers = TestHiveProtoLoggingHook.getTestReader(conf, tmpFolder);
+ Assert.assertEquals(1, readers.size());
+ ProtoMessageReader<HiveHookEvents.HiveHookEventProto> reader = readers.get(0);
HiveHookEvents.HiveHookEventProto event = reader.readEvent();
while (ExecutionMode.TEZ != ExecutionMode.valueOf(event.getExecutionMode())) {
event = reader.readEvent();
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
index 100a672f9b6..6b65f0c7873 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHiveProtoLoggingHook.java
@@ -22,8 +22,8 @@ import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -165,7 +165,9 @@ public class TestHiveProtoLoggingHook {
evtLogger.shutdown();
- ProtoMessageReader<HiveHookEventProto> reader = getTestReader(conf, tmpFolder);
+ List<ProtoMessageReader<HiveHookEventProto>> readers = getTestReader(conf, tmpFolder);
+ Assert.assertEquals(1, readers.size());
+ ProtoMessageReader<HiveHookEventProto> reader = readers.get(0);
HiveHookEventProto event = reader.readEvent();
Assert.assertNotNull(event);
@@ -193,7 +195,9 @@ public class TestHiveProtoLoggingHook {
evtLogger.handle(context);
evtLogger.handle(context);
evtLogger.shutdown();
- ProtoMessageReader<HiveHookEventProto> reader = getTestReader(conf, tmpFolder);
+ List<ProtoMessageReader<HiveHookEventProto>> readers = getTestReader(conf, tmpFolder);
+ Assert.assertEquals(1, readers.size());
+ ProtoMessageReader<HiveHookEventProto> reader = readers.get(0);
reader.readEvent();
reader.readEvent();
reader.readEvent();
@@ -212,7 +216,9 @@ public class TestHiveProtoLoggingHook {
evtLogger.handle(context);
evtLogger.shutdown();
- ProtoMessageReader<HiveHookEventProto> reader = getTestReader(conf, tmpFolder);
+ List<ProtoMessageReader<HiveHookEventProto>> readers = getTestReader(conf, tmpFolder);
+ Assert.assertEquals(1, readers.size());
+ ProtoMessageReader<HiveHookEventProto> reader = readers.get(0);
HiveHookEventProto event = reader.readEvent();
Assert.assertNotNull("Pre hook event not found", event);
Assert.assertEquals(EventType.QUERY_SUBMITTED.name(), event.getEventType());
@@ -302,21 +308,26 @@ public class TestHiveProtoLoggingHook {
Assert.assertEquals(2, statusLen);
}
- public static ProtoMessageReader<HiveHookEventProto> getTestReader(HiveConf conf, String tmpFolder)
+ public static List<ProtoMessageReader<HiveHookEventProto>> getTestReader(HiveConf conf, String tmpFolder)
throws IOException {
Path path = new Path(tmpFolder);
FileSystem fs = path.getFileSystem(conf);
- FileStatus[] status = fs.listStatus(path);
- Assert.assertEquals(1, status.length);
- status = fs.listStatus(status[0].getPath());
- Assert.assertEquals(1, status.length);
+ FileStatus[] folderStatuses = fs.listStatus(path);
DatePartitionedLogger<HiveHookEventProto> logger = new DatePartitionedLogger<>(
HiveHookEventProto.PARSER, path, conf, SystemClock.getInstance());
- return logger.getReader(status[0].getPath());
+ List<ProtoMessageReader<HiveHookEventProto>> readers = new ArrayList<>();
+ for (FileStatus folderStatus : folderStatuses) {
+ FileStatus[] status = fs.listStatus(folderStatus.getPath());
+ Assert.assertEquals(1, status.length);
+ readers.add(logger.getReader(status[0].getPath()));
+ }
+ return readers;
}
private HiveHookEventProto loadEvent(HiveConf conf, String tmpFolder) throws IOException {
- ProtoMessageReader<HiveHookEventProto> reader = getTestReader(conf, tmpFolder);
+ List<ProtoMessageReader<HiveHookEventProto>> readers = getTestReader(conf, tmpFolder);
+ Assert.assertEquals(1, readers.size());
+ ProtoMessageReader<HiveHookEventProto> reader = readers.get(0);
HiveHookEventProto event = reader.readEvent();
Assert.assertNotNull(event);
return event;