You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/08/22 22:58:26 UTC
[1/2] hbase git commit: HBASE-20648 HBASE-19364 "Truncate_preserve
fails with table when replica region > 1" for master branch
Repository: hbase
Updated Branches:
refs/heads/branch-2.0 e4c2d9fff -> 23556fb65
HBASE-20648 HBASE-19364 "Truncate_preserve fails with table when replica region > 1" for master branch
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/51eb02bb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/51eb02bb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/51eb02bb
Branch: refs/heads/branch-2.0
Commit: 51eb02bbe73e0b57f91519724fc0c1f51d752728
Parents: e4c2d9f
Author: Toshihiro Suzuki <br...@gmail.com>
Authored: Thu Aug 23 06:51:37 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Thu Aug 23 06:51:37 2018 +0800
----------------------------------------------------------------------
.../procedure/TruncateTableProcedure.java | 2 +
.../procedure/TestTruncateTableProcedure.java | 66 ++++++++++++++++++++
2 files changed, 68 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/51eb02bb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 57ea6e6..d6c8607 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
@@ -90,6 +91,7 @@ public class TruncateTableProcedure
// TODO: Move out... in the acquireLock()
LOG.debug("waiting for '" + getTableName() + "' regions in transition");
regions = env.getAssignmentManager().getRegionStates().getRegionsOfTable(getTableName());
+ RegionReplicaUtil.removeNonDefaultRegions(regions);
assert regions != null && !regions.isEmpty() : "unexpected 0 regions";
ProcedureSyncWait.waitRegionInTransition(env, regions);
http://git-wip-us.apache.org/repos/asf/hbase/blob/51eb02bb/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index d6d5421..bf5ae30 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -22,6 +22,8 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -30,7 +32,10 @@ import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
@@ -39,6 +44,7 @@ import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -46,6 +52,7 @@ import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
@Category({MasterTests.class, MediumTests.class})
@@ -307,4 +314,63 @@ public class TestTruncateTableProcedure extends TestTableDDLProcedureBase {
preserveSplits));
ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
}
+
+ @Test(timeout = 60000)
+ public void testTruncateWithPreserveAfterSplit() throws Exception {
+ String[] families = new String[] { "f1", "f2" };
+ byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
+ TableName tableName = TableName.valueOf(name.getMethodName());
+ RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
+ tableName, splitKeys, families);
+ splitAndTruncate(tableName, regions);
+ }
+
+ @Test(timeout = 60000)
+ public void testTruncatePreserveWithReplicaRegionAfterSplit() throws Exception {
+ String[] families = new String[] { "f1", "f2" };
+ byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
+ TableName tableName = TableName.valueOf(name.getMethodName());
+
+ // create a table with region replications
+ TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
+ .setRegionReplication(3)
+ .setColumnFamilies(
+ Arrays.stream(families)
+ .map(fam -> ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(fam)).build())
+ .collect(Collectors.toList()))
+ .build();
+ RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ long procId = ProcedureTestingUtility.submitAndWait(procExec,
+ new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
+ ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
+
+ splitAndTruncate(tableName, regions);
+ }
+
+ private void splitAndTruncate(TableName tableName, RegionInfo[] regions) throws IOException,
+ InterruptedException {
+
+ // split a region
+ UTIL.getAdmin().split(tableName, new byte[]{'0'});
+ UTIL.waitUntilAllRegionsAssigned(tableName);
+
+ // wait until split really happens
+ while (UTIL.getAdmin().getRegions(tableName).size() <= regions.length) {
+ Thread.sleep(50);
+ }
+
+ // disable the table
+ UTIL.getAdmin().disableTable(tableName);
+
+ // truncate the table
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ long procId = ProcedureTestingUtility.submitAndWait(procExec,
+ new TruncateTableProcedure(procExec.getEnvironment(), tableName, true));
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+
+ UTIL.waitUntilAllRegionsAssigned(tableName);
+ }
}
[2/2] hbase git commit: HBASE-20194 Remove the explicit timeout
config for TestTruncateTableProcedure
Posted by zh...@apache.org.
HBASE-20194 Remove the explicit timeout config for TestTruncateTableProcedure
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/23556fb6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/23556fb6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/23556fb6
Branch: refs/heads/branch-2.0
Commit: 23556fb655eb60df7020da168eb6a58ff9b73ac1
Parents: 51eb02b
Author: Duo Zhang <zh...@apache.org>
Authored: Wed Aug 22 11:29:23 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Thu Aug 23 06:57:14 2018 +0800
----------------------------------------------------------------------
.../procedure/TestTruncateTableProcedure.java | 37 ++++++++++----------
1 file changed, 18 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/23556fb6/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index bf5ae30..21557fb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -55,12 +55,12 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
-@Category({MasterTests.class, MediumTests.class})
+@Category({ MasterTests.class, MediumTests.class })
public class TestTruncateTableProcedure extends TestTableDDLProcedureBase {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestTruncateTableProcedure.class);
+ HBaseClassTestRule.forClass(TestTruncateTableProcedure.class);
private static final Logger LOG = LoggerFactory.getLogger(TestTruncateTableProcedure.class);
@@ -315,7 +315,7 @@ public class TestTruncateTableProcedure extends TestTableDDLProcedureBase {
ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
}
- @Test(timeout = 60000)
+ @Test
public void testTruncateWithPreserveAfterSplit() throws Exception {
String[] families = new String[] { "f1", "f2" };
byte[][] splitKeys =
@@ -323,10 +323,10 @@ public class TestTruncateTableProcedure extends TestTableDDLProcedureBase {
TableName tableName = TableName.valueOf(name.getMethodName());
RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
tableName, splitKeys, families);
- splitAndTruncate(tableName, regions);
+ splitAndTruncate(tableName, regions, 1);
}
- @Test(timeout = 60000)
+ @Test
public void testTruncatePreserveWithReplicaRegionAfterSplit() throws Exception {
String[] families = new String[] { "f1", "f2" };
byte[][] splitKeys =
@@ -334,12 +334,10 @@ public class TestTruncateTableProcedure extends TestTableDDLProcedureBase {
TableName tableName = TableName.valueOf(name.getMethodName());
// create a table with region replications
- TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
- .setRegionReplication(3)
- .setColumnFamilies(
- Arrays.stream(families)
- .map(fam -> ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(fam)).build())
- .collect(Collectors.toList()))
+ TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(3)
+ .setColumnFamilies(Arrays.stream(families)
+ .map(fam -> ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(fam)).build())
+ .collect(Collectors.toList()))
.build();
RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
@@ -347,20 +345,18 @@ public class TestTruncateTableProcedure extends TestTableDDLProcedureBase {
new CreateTableProcedure(procExec.getEnvironment(), htd, regions));
ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
- splitAndTruncate(tableName, regions);
+ splitAndTruncate(tableName, regions, 3);
}
- private void splitAndTruncate(TableName tableName, RegionInfo[] regions) throws IOException,
- InterruptedException {
-
+ private void splitAndTruncate(TableName tableName, RegionInfo[] regions, int regionReplication)
+ throws IOException, InterruptedException {
// split a region
- UTIL.getAdmin().split(tableName, new byte[]{'0'});
+ UTIL.getAdmin().split(tableName, new byte[] { '0' });
UTIL.waitUntilAllRegionsAssigned(tableName);
// wait until split really happens
- while (UTIL.getAdmin().getRegions(tableName).size() <= regions.length) {
- Thread.sleep(50);
- }
+ UTIL.waitFor(60000,
+ () -> UTIL.getAdmin().getRegions(tableName).size() > regions.length * regionReplication);
// disable the table
UTIL.getAdmin().disableTable(tableName);
@@ -372,5 +368,8 @@ public class TestTruncateTableProcedure extends TestTableDDLProcedureBase {
ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
UTIL.waitUntilAllRegionsAssigned(tableName);
+ // confirm that we have the correct number of regions
+ assertEquals((regions.length + 1) * regionReplication,
+ UTIL.getAdmin().getRegions(tableName).size());
}
}