You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2021/06/09 15:21:13 UTC
[ozone] branch master updated: HDDS-5312. Intermittent failure in
SCM Ratis integration test (#2308)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2e385fc HDDS-5312. Intermittent failure in SCM Ratis integration test (#2308)
2e385fc is described below
commit 2e385fcfb79877579ec1de639fe32f55c4907954
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Wed Jun 9 17:20:56 2021 +0200
HDDS-5312. Intermittent failure in SCM Ratis integration test (#2308)
---
.../hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java | 10 ++++++++--
.../org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java | 1 +
.../test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java | 1 +
.../java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java | 8 +++++---
.../org/apache/hadoop/ozone/TestStorageContainerManager.java | 1 +
.../apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java | 1 +
6 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java
index 4901a57..2ab0b31 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreatorV2.java
@@ -150,7 +150,7 @@ public class BackgroundPipelineCreatorV2 implements SCMService {
* Stop RatisPipelineUtilsThread.
*/
public void stop() {
- if (running.compareAndSet(true, false)) {
+ if (!running.compareAndSet(true, false)) {
LOG.warn("{} is not running, just ignore.", THREAD_NAME);
return;
}
@@ -280,7 +280,13 @@ public class BackgroundPipelineCreatorV2 implements SCMService {
|| event == UNHEALTHY_TO_HEALTHY_NODE_HANDLER_TRIGGERED
|| event == PRE_CHECK_COMPLETED) {
LOG.info("trigger a one-shot run on {}.", THREAD_NAME);
- oneShotRun = true;
+
+ serviceLock.lock();
+ try {
+ oneShotRun = true;
+ } finally {
+ serviceLock.unlock();
+ }
synchronized (monitor) {
monitor.notifyAll();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
index 6765450..83f51ac 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshot.java
@@ -62,6 +62,7 @@ public class TestSCMInstallSnapshot {
public static void setup() throws Exception {
conf = new OzoneConfiguration();
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+ conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
SCMHAConfiguration scmhaConfiguration = conf.getObject(
SCMHAConfiguration.class);
scmhaConfiguration.setRatisSnapshotThreshold(1L);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java
index 275df6d..84b9e58 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java
@@ -48,6 +48,7 @@ public class TestSCMSnapshot {
public static void setup() throws Exception {
conf = new OzoneConfiguration();
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+ conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
SCMHAConfiguration scmhaConfiguration = conf.getObject(
SCMHAConfiguration.class);
scmhaConfiguration.setRatisSnapshotThreshold(1L);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 27ceab0..b09859f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -200,7 +200,10 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
public void waitForSCMToBeReady() throws TimeoutException,
InterruptedException {
- // Nothing implemented here
+ if (SCMHAUtils.isSCMHAEnabled(conf)) {
+ GenericTestUtils.waitFor(scm::checkLeader,
+ 1000, waitForClusterToBeReadyTimeout);
+ }
}
public StorageContainerManager getActiveSCM() {
@@ -225,8 +228,7 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster {
isNodeReady ? "Nodes are ready" : "Waiting for nodes to be ready",
healthy, hddsDatanodes.size());
LOG.info(exitSafeMode ? "Cluster exits safe mode" :
- "Waiting for cluster to exit safe mode",
- healthy, hddsDatanodes.size());
+ "Waiting for cluster to exit safe mode");
LOG.info(checkScmLeader ? "SCM became leader" :
"SCM has not become leader");
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index e39884d..7763855 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -493,6 +493,7 @@ public class TestStorageContainerManager {
public void testSCMInitializationWithHAEnabled() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
+ conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
final String path = GenericTestUtils.getTempPath(
UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
index 995949f..a5d5505 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestStorageContainerManagerHA.java
@@ -88,6 +88,7 @@ public class TestStorageContainerManagerHA {
@Before
public void init() throws Exception {
conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
omServiceId = "om-service-test1";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org