You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/04/22 01:22:42 UTC
(hbase) branch branch-2.5 updated: HBASE-28150 CreateTableProcedure and DeleteTableProcedure should sleep a while before retrying (#5842)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 4d7ce1aac72 HBASE-28150 CreateTableProcedure and DeleteTableProcedure should sleep a while before retrying (#5842)
4d7ce1aac72 is described below
commit 4d7ce1aac724fbf09e526fc422b5a11e530c32f0
Author: chaijunjie0101 <64...@users.noreply.github.com>
AuthorDate: Mon Apr 22 09:12:59 2024 +0800
HBASE-28150 CreateTableProcedure and DeleteTableProcedure should sleep a while before retrying (#5842)
Conflict: Copy suspend method from HBASE-27218, it is a great method to process suspend, no need write same code again into branch2, so if we want backport HBASE-27218, could skip this method.
Signed-off-by: Duo Zhang <zh...@apache.org>
(cherry picked from commit a36da5ef8607ff0915b26c371a3dfec6e03ad610)
---
.../apache/hadoop/hbase/procedure2/Procedure.java | 15 ++++
.../master/procedure/CreateTableProcedure.java | 24 +++++-
.../master/procedure/DeleteTableProcedure.java | 24 +++++-
.../BadMasterObserverForCreateDeleteTable.java | 55 ++++++++++++++
.../TestCreateDeleteTableProcedureWithRetry.java | 88 ++++++++++++++++++++++
.../master/procedure/TestCreateTableProcedure.java | 4 +-
6 files changed, 205 insertions(+), 5 deletions(-)
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index 44639af1a9f..3cf3b2f5ee5 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
import org.apache.hadoop.hbase.metrics.Counter;
import org.apache.hadoop.hbase.metrics.Histogram;
@@ -33,6 +34,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
/**
@@ -1034,6 +1036,19 @@ public abstract class Procedure<TEnvironment> implements Comparable<Procedure<TE
releaseLock(env);
}
+ protected final ProcedureSuspendedException suspend(int timeoutMillis, boolean jitter)
+ throws ProcedureSuspendedException {
+ if (jitter) {
+ // 10% possible jitter
+ double add = (double) timeoutMillis * ThreadLocalRandom.current().nextDouble(0.1);
+ timeoutMillis += add;
+ }
+ setTimeout(timeoutMillis);
+ setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
+ skipPersistence();
+ throw new ProcedureSuspendedException();
+ }
+
@Override
public int compareTo(final Procedure<TEnvironment> other) {
return Long.compare(getProcId(), other.getProcId());
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
index 08be7238821..553eecf71a8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
@@ -33,12 +33,15 @@ import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerValidationUtils;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
+import org.apache.hadoop.hbase.util.RetryCounter;
import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -50,6 +53,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CreateTableState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
@InterfaceAudience.Private
public class CreateTableProcedure extends AbstractStateMachineTableProcedure<CreateTableState> {
@@ -59,6 +63,7 @@ public class CreateTableProcedure extends AbstractStateMachineTableProcedure<Cre
private TableDescriptor tableDescriptor;
private List<RegionInfo> newRegions;
+ private RetryCounter retryCounter;
public CreateTableProcedure() {
// Required by the Procedure framework to create the procedure on replay
@@ -79,7 +84,7 @@ public class CreateTableProcedure extends AbstractStateMachineTableProcedure<Cre
@Override
protected Flow executeFromState(final MasterProcedureEnv env, final CreateTableState state)
- throws InterruptedException {
+ throws InterruptedException, ProcedureSuspendedException {
LOG.info("{} execute state={}", this, state);
try {
switch (state) {
@@ -120,6 +125,7 @@ public class CreateTableProcedure extends AbstractStateMachineTableProcedure<Cre
break;
case CREATE_TABLE_POST_OPERATION:
postCreate(env);
+ retryCounter = null;
return Flow.NO_MORE_STATE;
default:
throw new UnsupportedOperationException("unhandled state=" + state);
@@ -128,12 +134,26 @@ public class CreateTableProcedure extends AbstractStateMachineTableProcedure<Cre
if (isRollbackSupported(state)) {
setFailure("master-create-table", e);
} else {
- LOG.warn("Retriable error trying to create table=" + getTableName() + " state=" + state, e);
+ if (retryCounter == null) {
+ retryCounter = ProcedureUtil.createRetryCounter(env.getMasterConfiguration());
+ }
+ long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
+ LOG.warn("Retriable error trying to create table={},state={},suspend {}secs.",
+ getTableName(), state, backoff / 1000, e);
+ throw suspend(Math.toIntExact(backoff), true);
}
}
+ retryCounter = null;
return Flow.HAS_MORE_STATE;
}
+ @Override
+ protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
+ setState(ProcedureProtos.ProcedureState.RUNNABLE);
+ env.getProcedureScheduler().addFront(this);
+ return false;
+ }
+
@Override
protected void rollbackState(final MasterProcedureEnv env, final CreateTableState state)
throws IOException {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
index 95aab682e22..da6ad90780d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java
@@ -41,9 +41,12 @@ import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.RetryCounter;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,6 +57,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.DeleteTableState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
@InterfaceAudience.Private
public class DeleteTableProcedure extends AbstractStateMachineTableProcedure<DeleteTableState> {
@@ -61,6 +65,7 @@ public class DeleteTableProcedure extends AbstractStateMachineTableProcedure<Del
private List<RegionInfo> regions;
private TableName tableName;
+ private RetryCounter retryCounter;
public DeleteTableProcedure() {
// Required by the Procedure framework to create the procedure on replay
@@ -79,7 +84,7 @@ public class DeleteTableProcedure extends AbstractStateMachineTableProcedure<Del
@Override
protected Flow executeFromState(final MasterProcedureEnv env, DeleteTableState state)
- throws InterruptedException {
+ throws InterruptedException, ProcedureSuspendedException {
if (LOG.isTraceEnabled()) {
LOG.trace(this + " execute state=" + state);
}
@@ -124,6 +129,7 @@ public class DeleteTableProcedure extends AbstractStateMachineTableProcedure<Del
break;
case DELETE_TABLE_POST_OPERATION:
postDelete(env);
+ retryCounter = null;
LOG.debug("Finished {}", this);
return Flow.NO_MORE_STATE;
default:
@@ -133,12 +139,26 @@ public class DeleteTableProcedure extends AbstractStateMachineTableProcedure<Del
if (isRollbackSupported(state)) {
setFailure("master-delete-table", e);
} else {
- LOG.warn("Retriable error trying to delete table=" + getTableName() + " state=" + state, e);
+ if (retryCounter == null) {
+ retryCounter = ProcedureUtil.createRetryCounter(env.getMasterConfiguration());
+ }
+ long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
+ LOG.warn("Retriable error trying to delete table={},state={},suspend {}secs.",
+ getTableName(), state, backoff / 1000, e);
+ throw suspend(Math.toIntExact(backoff), true);
}
}
+ retryCounter = null;
return Flow.HAS_MORE_STATE;
}
+ @Override
+ protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
+ setState(ProcedureProtos.ProcedureState.RUNNABLE);
+ env.getProcedureScheduler().addFront(this);
+ return false;
+ }
+
@Override
protected boolean abort(MasterProcedureEnv env) {
// TODO: Current behavior is: with no rollback and no abort support, procedure may get stuck
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/BadMasterObserverForCreateDeleteTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/BadMasterObserverForCreateDeleteTable.java
new file mode 100644
index 00000000000..454a24e198a
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/BadMasterObserverForCreateDeleteTable.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+import java.util.Optional;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+
+/**
+ * A bad Master Observer to prevent user to create/delete table once.
+ */
+public class BadMasterObserverForCreateDeleteTable implements MasterObserver, MasterCoprocessor {
+ private boolean createFailedOnce = false;
+ private boolean deleteFailedOnce = false;
+
+ @Override
+ public void postCompletedCreateTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableDescriptor desc, RegionInfo[] regions) throws IOException {
+ if (!createFailedOnce && !desc.getTableName().isSystemTable()) {
+ createFailedOnce = true;
+ throw new IOException("execute postCompletedCreateTableAction failed once.");
+ }
+ }
+
+ @Override
+ public void postCompletedDeleteTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName) throws IOException {
+ if (!deleteFailedOnce && !tableName.isSystemTable()) {
+ deleteFailedOnce = true;
+ throw new IOException("execute postCompletedDeleteTableAction failed once.");
+ }
+ }
+
+ @Override
+ public Optional<MasterObserver> getMasterObserver() {
+ return Optional.of(this);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateDeleteTableProcedureWithRetry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateDeleteTableProcedureWithRetry.java
new file mode 100644
index 00000000000..5cee4f70f10
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateDeleteTableProcedureWithRetry.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.coprocessor.BadMasterObserverForCreateDeleteTable;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.ModifyRegionUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestCreateDeleteTableProcedureWithRetry {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestCreateDeleteTableProcedureWithRetry.class);
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static final TableName TABLE_NAME =
+ TableName.valueOf(TestCreateDeleteTableProcedureWithRetry.class.getSimpleName());
+
+ private static final String CF = "cf";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Configuration conf = UTIL.getConfiguration();
+ conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+ BadMasterObserverForCreateDeleteTable.class.getName());
+ UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testCreateDeleteTableRetry() throws IOException {
+ ProcedureExecutor<MasterProcedureEnv> procExec =
+ UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
+ TableDescriptor htd = MasterProcedureTestingUtility.createHTD(TABLE_NAME, CF);
+ RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null);
+ CreateTableProcedure createProc =
+ new CreateTableProcedure(procExec.getEnvironment(), htd, regions);
+ ProcedureTestingUtility.submitAndWait(procExec, createProc);
+ Assert.assertTrue(UTIL.getAdmin().tableExists(TABLE_NAME));
+ MasterProcedureTestingUtility.validateTableCreation(UTIL.getMiniHBaseCluster().getMaster(),
+ TABLE_NAME, regions, CF);
+
+ UTIL.getAdmin().disableTable(TABLE_NAME);
+ DeleteTableProcedure deleteProc =
+ new DeleteTableProcedure(procExec.getEnvironment(), TABLE_NAME);
+ ProcedureTestingUtility.submitAndWait(procExec, deleteProc);
+ Assert.assertFalse(UTIL.getAdmin().tableExists(TABLE_NAME));
+ MasterProcedureTestingUtility.validateTableDeletion(UTIL.getMiniHBaseCluster().getMaster(),
+ TABLE_NAME);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java
index bed41f4da86..0bb54cc190e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerForTest;
@@ -244,7 +245,8 @@ public class TestCreateTableProcedure extends TestTableDDLProcedureBase {
@Override
protected Flow executeFromState(MasterProcedureEnv env,
- MasterProcedureProtos.CreateTableState state) throws InterruptedException {
+ MasterProcedureProtos.CreateTableState state)
+ throws InterruptedException, ProcedureSuspendedException {
if (
!failOnce && state == MasterProcedureProtos.CreateTableState.CREATE_TABLE_WRITE_FS_LAYOUT