You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2018/05/25 13:18:52 UTC
hbase git commit: HBASE-20647 Backport HBASE-20616
"TruncateTableProcedure is stuck in retry loop in
TRUNCATE_TABLE_CREATE_FS_LAYOUT state" to branch-1
Repository: hbase
Updated Branches:
refs/heads/branch-1 87d4e4e4e -> 289d588af
HBASE-20647 Backport HBASE-20616 "TruncateTableProcedure is stuck in retry loop in TRUNCATE_TABLE_CREATE_FS_LAYOUT state" to branch-1
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/289d588a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/289d588a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/289d588a
Branch: refs/heads/branch-1
Commit: 289d588af32c8aa94758d2e4b299eb40a2ecc6ac
Parents: 87d4e4e
Author: Toshihiro Suzuki <br...@gmail.com>
Authored: Fri May 25 09:37:19 2018 +0900
Committer: tedyu <yu...@gmail.com>
Committed: Fri May 25 06:18:43 2018 -0700
----------------------------------------------------------------------
.../procedure/TruncateTableProcedure.java | 35 ++++---
.../procedure/TestTruncateTableProcedure.java | 101 +++++++++++++++++--
2 files changed, 117 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/289d588a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
index 120ae4f..4689426 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java
@@ -18,29 +18,29 @@
package org.apache.hadoop.hbase.master.procedure;
-import java.io.InputStream;
+import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.exceptions.HBaseException;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.TruncateTableState;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -103,16 +103,15 @@ public class TruncateTableProcedure
case TRUNCATE_TABLE_CLEAR_FS_LAYOUT:
DeleteTableProcedure.deleteFromFs(env, getTableName(), regions, true);
setNextState(TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT);
- break;
- case TRUNCATE_TABLE_CREATE_FS_LAYOUT:
if (!preserveSplits) {
// if we are not preserving splits, generate a new single region
- //recreateRegionInfo in TRUNCATE_TABLE_CREATE_FS_LAYOUT phase, since if create fs layout fails
- //we need to refresh the region encoded name to prevent dir name conflict
regions = Arrays.asList(ModifyRegionUtils.createHRegionInfos(hTableDescriptor, null));
} else {
regions = recreateRegionInfo(regions);
}
+ break;
+ case TRUNCATE_TABLE_CREATE_FS_LAYOUT:
+ DeleteTableProcedure.deleteFromFs(env, getTableName(), regions, true);
regions = CreateTableProcedure.createFsLayout(env, hTableDescriptor, regions);
CreateTableProcedure.updateTableDescCache(env, getTableName());
setNextState(TruncateTableState.TRUNCATE_TABLE_ADD_TO_META);
@@ -185,7 +184,9 @@ public class TruncateTableProcedure
@Override
protected boolean acquireLock(final MasterProcedureEnv env) {
- if (env.waitInitialized(this)) return false;
+ if (env.waitInitialized(this)) {
+ return false;
+ }
return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, getTableName());
}
@@ -297,4 +298,12 @@ public class TruncateTableProcedure
});
}
}
-}
\ No newline at end of file
+
+ @VisibleForTesting
+ HRegionInfo getFirstRegionInfo() {
+ if (regions == null || regions.isEmpty()) {
+ return null;
+ }
+ return regions.get(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/289d588a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
index 47460af..6bc0256 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java
@@ -18,21 +18,33 @@
package org.apache.hadoop.hbase.master.procedure;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.ProcedureInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.TruncateTableState;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -40,11 +52,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-
@Category(MediumTests.class)
public class TestTruncateTableProcedure {
private static final Log LOG = LogFactory.getLog(TestTruncateTableProcedure.class);
@@ -289,4 +296,86 @@ public class TestTruncateTableProcedure {
UTIL.waitUntilAllRegionsAssigned(tableName);
}
+
+ @Test
+ public void testOnHDFSFailurePreserveSplits() throws Exception {
+ final TableName tableName = TableName.valueOf("testOnHDFSFailurePreserveSplits");
+ testOnHDFSFailure(tableName, true);
+ }
+
+ @Test
+ public void testOnHDFSFailureNoPreserveSplits() throws Exception {
+ final TableName tableName = TableName.valueOf("testOnHDFSFailureNoPreserveSplits");
+ testOnHDFSFailure(tableName, false);
+ }
+
+ public static class TruncateTableProcedureOnHDFSFailure extends TruncateTableProcedure {
+
+ private boolean failOnce = false;
+
+ public TruncateTableProcedureOnHDFSFailure() {
+ // Required by the Procedure framework to create the procedure on replay
+ super();
+ }
+
+ public TruncateTableProcedureOnHDFSFailure(final MasterProcedureEnv env, TableName tableName,
+ boolean preserveSplits)
+ throws HBaseIOException {
+ super(env, tableName, preserveSplits);
+ }
+
+ @Override
+ protected Flow executeFromState(MasterProcedureEnv env,
+ MasterProcedureProtos.TruncateTableState state) throws InterruptedException {
+
+ if (!failOnce &&
+ state == MasterProcedureProtos.TruncateTableState.TRUNCATE_TABLE_CREATE_FS_LAYOUT) {
+ try {
+ // To emulate an HDFS failure, create only the first region directory
+ HRegionInfo regionInfo = getFirstRegionInfo();
+ Configuration conf = env.getMasterConfiguration();
+ MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+ Path tempdir = mfs.getTempDir();
+ Path tableDir = FSUtils.getTableDir(tempdir, regionInfo.getTable());
+ Path regionDir = new Path(tableDir,
+ ServerRegionReplicaUtil.getRegionInfoForFs(regionInfo).getEncodedName());
+ FileSystem fs = FileSystem.get(conf);
+ fs.mkdirs(regionDir);
+
+ failOnce = true;
+ return Flow.HAS_MORE_STATE;
+ } catch (IOException e) {
+ fail("failed to create a region directory: " + e);
+ }
+ }
+
+ return super.executeFromState(env, state);
+ }
+ }
+
+ private void testOnHDFSFailure(TableName tableName, boolean preserveSplits) throws Exception {
+ String[] families = new String[] { "f1", "f2" };
+ byte[][] splitKeys = new byte[][] {
+ Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
+ };
+
+ // create a table
+ MasterProcedureTestingUtility.createTable(
+ getMasterProcedureExecutor(), tableName, splitKeys, families);
+
+ // load and verify that there are rows in the table
+ MasterProcedureTestingUtility.loadData(
+ UTIL.getConnection(), tableName, 100, splitKeys, families);
+ assertEquals(100, UTIL.countRows(tableName));
+
+ // disable the table
+ UTIL.getHBaseAdmin().disableTable(tableName);
+
+ // truncate the table
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ long procId = ProcedureTestingUtility.submitAndWait(procExec,
+ new TruncateTableProcedureOnHDFSFailure(procExec.getEnvironment(), tableName,
+ preserveSplits));
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+ }
}