You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2020/05/26 18:58:01 UTC
[hbase] branch master updated: HBASE-24437 Flaky test,
TestLocalRegionOnTwoFileSystems#testFlushAndCompact
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new a9205f8 HBASE-24437 Flaky test, TestLocalRegionOnTwoFileSystems#testFlushAndCompact
a9205f8 is described below
commit a9205f8f4d98ee672c0c6aa9cafa5ef2afc6aab5
Author: stack <st...@apache.org>
AuthorDate: Tue May 26 11:52:47 2020 -0700
HBASE-24437 Flaky test, TestLocalRegionOnTwoFileSystems#testFlushAndCompact
Force another WAL roll just in case and make the check more loose (its
about whether the files are present, not their count)
---
.../store/TestLocalRegionOnTwoFileSystems.java | 37 ++++++++++++++++------
1 file changed, 27 insertions(+), 10 deletions(-)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/store/TestLocalRegionOnTwoFileSystems.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/store/TestLocalRegionOnTwoFileSystems.java
index 94cfe40..0aa1362 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/store/TestLocalRegionOnTwoFileSystems.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/store/TestLocalRegionOnTwoFileSystems.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -22,12 +22,13 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -53,6 +54,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
+import org.apache.hadoop.hbase.util.Threads;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -60,11 +62,13 @@ import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
@Category({ MasterTests.class, MediumTests.class })
public class TestLocalRegionOnTwoFileSystems {
+ private static final Logger LOG = LoggerFactory.getLogger(TestLocalRegionOnTwoFileSystems.class);
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
@@ -78,8 +82,6 @@ public class TestLocalRegionOnTwoFileSystems {
private static byte[] CQ = Bytes.toBytes("q");
- private static String REGION_DIR_NAME = "local";
-
private static TableDescriptor TD =
TableDescriptorBuilder.newBuilder(TableName.valueOf("test:local"))
.setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build();
@@ -114,7 +116,7 @@ public class TestLocalRegionOnTwoFileSystems {
when(server.getConfiguration()).thenReturn(HFILE_UTIL.getConfiguration());
when(server.getServerName()).thenReturn(serverName);
LocalRegionParams params = new LocalRegionParams();
- params.server(server).regionDirName(REGION_DIR_NAME).tableDescriptor(TD)
+ params.server(server).regionDirName("local").tableDescriptor(TD)
.flushSize(TableDescriptorBuilder.DEFAULT_MEMSTORE_FLUSH_SIZE).flushPerChanges(1_000_000)
.flushIntervalMs(TimeUnit.MINUTES.toMillis(15)).compactMin(COMPACT_MIN).maxWals(32)
.useHsync(false).ringBufferSlotCount(16).rollPeriodMs(TimeUnit.MINUTES.toMillis(15))
@@ -145,7 +147,8 @@ public class TestLocalRegionOnTwoFileSystems {
@Test
public void testFlushAndCompact() throws Exception {
- for (int i = 0; i < COMPACT_MIN - 1; i++) {
+ int compactMinMinusOne = COMPACT_MIN - 1;
+ for (int i = 0; i < compactMinMinusOne; i++) {
final int index = i;
region
.update(r -> r.put(new Put(Bytes.toBytes(index)).addColumn(CF, CQ, Bytes.toBytes(index))));
@@ -153,8 +156,8 @@ public class TestLocalRegionOnTwoFileSystems {
}
region.requestRollAll();
region.waitUntilWalRollFinished();
- region.update(r -> r.put(
- new Put(Bytes.toBytes(COMPACT_MIN - 1)).addColumn(CF, CQ, Bytes.toBytes(COMPACT_MIN - 1))));
+ byte [] bytes = Bytes.toBytes(compactMinMinusOne);
+ region.update(r -> r.put(new Put(bytes).addColumn(CF, CQ, bytes)));
region.flusherAndCompactor.requestFlush();
HFILE_UTIL.waitFor(15000, () -> getStorefilesCount() == 1);
@@ -171,14 +174,28 @@ public class TestLocalRegionOnTwoFileSystems {
return false;
}
});
+ LOG.info("hfile archive content {}",
+ Arrays.stream(rootFs.listStatus(storeArchiveDir)).map(f -> f.getPath().toString()).
+ collect(Collectors.joining(",")));
// make sure the archived wal files are on the wal fs
Path walArchiveDir = new Path(CommonFSUtils.getWALRootDir(HFILE_UTIL.getConfiguration()),
HConstants.HREGION_OLDLOGDIR_NAME);
+ LOG.info("wal archive dir {}", walArchiveDir);
+ region.requestRollAll();
+ region.waitUntilWalRollFinished();
HFILE_UTIL.waitFor(15000, () -> {
try {
FileStatus[] fses = WAL_UTIL.getTestFileSystem().listStatus(walArchiveDir);
- return fses != null && fses.length == 1;
+ if (fses != null) {
+ LOG.info("wal archive dir content {}",
+ Arrays.stream(fses).map(f -> f.getPath().toString()).
+ collect(Collectors.joining(",")));
+ } else {
+ LOG.info("none found");
+ Threads.sleep(100);
+ }
+ return fses != null && fses.length >= 1;
} catch (FileNotFoundException e) {
return false;
}