You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ad...@apache.org on 2022/05/04 21:23:11 UTC
[ozone] branch master updated: HDDS-6655. Use injected clocks in PipelineManagerImpl and BackgroundPipelineScrubber to ease testing (#3363)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9ff16ac551 HDDS-6655. Use injected clocks in PipelineManagerImpl and BackgroundPipelineScrubber to ease testing (#3363)
9ff16ac551 is described below
commit 9ff16ac551a58337b94a6bd546223f454aca08e8
Author: Stephen O'Donnell <st...@gmail.com>
AuthorDate: Wed May 4 22:23:06 2022 +0100
HDDS-6655. Use injected clocks in PipelineManagerImpl and BackgroundPipelineScrubber to ease testing (#3363)
---
.../scm/pipeline/BackgroundPipelineCreator.java | 10 +++---
.../scm/pipeline/BackgroundPipelineScrubber.java | 18 +++++-----
.../hdds/scm/pipeline/PipelineManagerImpl.java | 26 +++++++++-----
.../hdds/scm/server/StorageContainerManager.java | 8 +++--
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +++-
.../container/TestCloseContainerEventHandler.java | 6 +++-
.../pipeline/TestBackgroundPipelineScrubber.java | 16 +++++++--
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 41 ++++++++++++++++------
.../safemode/TestHealthyPipelineSafeModeRule.java | 11 ++++--
.../TestOneReplicaPipelineSafeModeRule.java | 6 +++-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +++++++----
.../ozone/recon/scm/ReconPipelineManager.java | 5 ++-
12 files changed, 123 insertions(+), 50 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
index 006453fad3..2b97eaeb47 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineCreator.java
@@ -30,11 +30,11 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMService;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.time.Clock;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -84,13 +84,15 @@ public class BackgroundPipelineCreator implements SCMService {
private static final String THREAD_NAME = "RatisPipelineUtilsThread";
private final AtomicBoolean running = new AtomicBoolean(false);
private final long intervalInMillis;
+ private final Clock clock;
BackgroundPipelineCreator(PipelineManager pipelineManager,
- ConfigurationSource conf, SCMContext scmContext) {
+ ConfigurationSource conf, SCMContext scmContext, Clock clock) {
this.pipelineManager = pipelineManager;
this.conf = conf;
this.scmContext = scmContext;
+ this.clock = clock;
this.createPipelineInSafeMode = conf.getBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION,
@@ -247,7 +249,7 @@ public class BackgroundPipelineCreator implements SCMService {
// transition from PAUSING to RUNNING
if (serviceStatus != ServiceStatus.RUNNING) {
LOG.info("Service {} transitions to RUNNING.", getServiceName());
- lastTimeToBeReadyInMillis = Time.monotonicNow();
+ lastTimeToBeReadyInMillis = clock.millis();
serviceStatus = ServiceStatus.RUNNING;
}
} else {
@@ -295,7 +297,7 @@ public class BackgroundPipelineCreator implements SCMService {
// If safe mode is off, then this SCMService starts to run with a delay.
return serviceStatus == ServiceStatus.RUNNING && (
createPipelineInSafeMode ||
- Time.monotonicNow() - lastTimeToBeReadyInMillis >= waitTimeInMillis);
+ clock.millis() - lastTimeToBeReadyInMillis >= waitTimeInMillis);
} finally {
serviceLock.unlock();
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineScrubber.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineScrubber.java
index 29fbd00d7c..3dc5f4177f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineScrubber.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/BackgroundPipelineScrubber.java
@@ -23,11 +23,11 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.SCMService;
-import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.time.Clock;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
@@ -58,12 +58,14 @@ public class BackgroundPipelineScrubber implements SCMService {
private final long waitTimeInMillis;
private long lastTimeToBeReadyInMillis = 0;
private volatile boolean runImmediately = false;
+ private final Clock clock;
public BackgroundPipelineScrubber(PipelineManager pipelineManager,
- ConfigurationSource conf, SCMContext scmContext) {
+ ConfigurationSource conf, SCMContext scmContext, Clock clock) {
this.pipelineManager = pipelineManager;
this.conf = conf;
this.scmContext = scmContext;
+ this.clock = clock;
this.intervalInMillis = conf.getTimeDuration(
ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL,
@@ -85,7 +87,7 @@ public class BackgroundPipelineScrubber implements SCMService {
if (serviceStatus != ServiceStatus.RUNNING) {
LOG.info("Service {} transitions to RUNNING.", getServiceName());
serviceStatus = ServiceStatus.RUNNING;
- lastTimeToBeReadyInMillis = Time.monotonicNow();
+ lastTimeToBeReadyInMillis = clock.millis();
}
} else {
if (serviceStatus != ServiceStatus.PAUSING) {
@@ -104,7 +106,7 @@ public class BackgroundPipelineScrubber implements SCMService {
try {
// If safe mode is off, then this SCMService starts to run with a delay.
return serviceStatus == ServiceStatus.RUNNING &&
- Time.monotonicNow() - lastTimeToBeReadyInMillis >= waitTimeInMillis;
+ clock.millis() - lastTimeToBeReadyInMillis >= waitTimeInMillis;
} finally {
serviceLock.unlock();
}
@@ -118,10 +120,10 @@ public class BackgroundPipelineScrubber implements SCMService {
@Override
public void start() {
if (!running.compareAndSet(false, true)) {
- LOG.info("Pipeline Scrubber Service is already running, skip start.");
+ LOG.info("{} Service is already running, skip start.", getServiceName());
return;
}
- LOG.info("Starting Pipeline Scrubber Service.");
+ LOG.info("Starting {} Service.", getServiceName());
scrubThread = new Thread(this::run);
scrubThread.setName(THREAD_NAME);
@@ -133,12 +135,12 @@ public class BackgroundPipelineScrubber implements SCMService {
public void stop() {
synchronized (this) {
if (!running.compareAndSet(true, false)) {
- LOG.info("Pipeline Scrubber Service is not running, skip stop.");
+ LOG.info("{} Service is not running, skip stop.", getServiceName());
return;
}
scrubThread.interrupt();
}
- LOG.info("Stopping Pipeline Scrubber Service.");
+ LOG.info("Stopping {} Service.", getServiceName());
}
@VisibleForTesting
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java
index 83b037cb2f..1cc4b45261 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerImpl.java
@@ -42,13 +42,13 @@ import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
-import org.apache.hadoop.util.Time;
import org.apache.ratis.protocol.exceptions.NotLeaderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.management.ObjectName;
import java.io.IOException;
+import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.Collection;
@@ -89,14 +89,17 @@ public class PipelineManagerImpl implements PipelineManager {
// This allows for freezing/resuming the new pipeline creation while the
// SCM is already out of SafeMode.
private AtomicBoolean freezePipelineCreation;
+ private final Clock clock;
+ @SuppressWarnings("checkstyle:parameterNumber")
protected PipelineManagerImpl(ConfigurationSource conf,
SCMHAManager scmhaManager,
NodeManager nodeManager,
PipelineStateManager pipelineStateManager,
PipelineFactory pipelineFactory,
EventPublisher eventPublisher,
- SCMContext scmContext) {
+ SCMContext scmContext,
+ Clock clock) {
this.lock = new ReentrantReadWriteLock();
this.pipelineFactory = pipelineFactory;
this.stateManager = pipelineStateManager;
@@ -105,6 +108,7 @@ public class PipelineManagerImpl implements PipelineManager {
this.nodeManager = nodeManager;
this.eventPublisher = eventPublisher;
this.scmContext = scmContext;
+ this.clock = clock;
this.pmInfoBean = MBeans.register("SCMPipelineManager",
"SCMPipelineManagerInfo", this);
this.metrics = SCMPipelineMetrics.create();
@@ -115,6 +119,7 @@ public class PipelineManagerImpl implements PipelineManager {
this.freezePipelineCreation = new AtomicBoolean();
}
+ @SuppressWarnings("checkstyle:parameterNumber")
public static PipelineManagerImpl newPipelineManager(
ConfigurationSource conf,
SCMHAManager scmhaManager,
@@ -122,7 +127,9 @@ public class PipelineManagerImpl implements PipelineManager {
Table<PipelineID, Pipeline> pipelineStore,
EventPublisher eventPublisher,
SCMContext scmContext,
- SCMServiceManager serviceManager) throws IOException {
+ SCMServiceManager serviceManager,
+ Clock clock) throws IOException {
+
// Create PipelineStateManagerImpl
PipelineStateManager stateManager = PipelineStateManagerImpl
.newBuilder().setPipelineStore(pipelineStore)
@@ -138,14 +145,15 @@ public class PipelineManagerImpl implements PipelineManager {
// Create PipelineManager
PipelineManagerImpl pipelineManager = new PipelineManagerImpl(conf,
scmhaManager, nodeManager, stateManager, pipelineFactory,
- eventPublisher, scmContext);
+ eventPublisher, scmContext, clock);
// Create background thread.
BackgroundPipelineCreator backgroundPipelineCreator =
- new BackgroundPipelineCreator(pipelineManager, conf, scmContext);
+ new BackgroundPipelineCreator(pipelineManager, conf, scmContext, clock);
BackgroundPipelineScrubber backgroundPipelineScrubber =
- new BackgroundPipelineScrubber(pipelineManager, conf, scmContext);
+ new BackgroundPipelineScrubber(pipelineManager, conf, scmContext,
+ clock);
pipelineManager.setBackgroundPipelineCreator(backgroundPipelineCreator);
pipelineManager.setBackgroundPipelineScrubber(backgroundPipelineScrubber);
@@ -416,7 +424,7 @@ public class PipelineManagerImpl implements PipelineManager {
*/
@Override
public void scrubPipelines() throws IOException {
- Instant currentTime = Instant.now();
+ Instant currentTime = clock.instant();
Long pipelineScrubTimeoutInMills = conf.getTimeDuration(
ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT,
ScmConfigKeys.OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT_DEFAULT,
@@ -522,7 +530,7 @@ public class PipelineManagerImpl implements PipelineManager {
@Override
public void waitPipelineReady(PipelineID pipelineID, long timeout)
throws IOException {
- long st = Time.monotonicNow();
+ long st = clock.millis();
if (timeout == 0) {
timeout = pipelineWaitDefaultTimeout;
}
@@ -544,7 +552,7 @@ public class PipelineManagerImpl implements PipelineManager {
Thread.currentThread().interrupt();
}
}
- } while (!ready && Time.monotonicNow() - st < timeout);
+ } while (!ready && clock.millis() - st < timeout);
if (!ready) {
throw new IOException(String.format("Pipeline %s is not ready in %d ms",
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 9abe25969b..13b3f7239b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -154,6 +154,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
+import java.time.Clock;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Collection;
@@ -552,6 +553,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private void initializeSystemManagers(OzoneConfiguration conf,
SCMConfigurator configurator)
throws IOException {
+ Clock clock = new MonotonicClock(ZoneOffset.UTC);
if (configurator.getNetworkTopology() != null) {
clusterMap = configurator.getNetworkTopology();
} else {
@@ -610,7 +612,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmMetadataStore.getPipelineTable(),
eventQueue,
scmContext,
- serviceManager);
+ serviceManager,
+ clock
+ );
}
if (configurator.getContainerManager() != null) {
@@ -642,7 +646,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmContext,
serviceManager,
scmNodeManager,
- new MonotonicClock(ZoneOffset.UTC),
+ clock,
scmHAManager,
getScmMetadataStore().getMoveTable());
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
index 82834ecf99..5aa3206c3b 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.block;
import java.io.IOException;
+import java.time.ZoneOffset;
import java.util.List;
import java.util.Map;
import java.util.ArrayList;
@@ -66,6 +67,7 @@ import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.common.MonotonicClock;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand;
@@ -73,6 +75,7 @@ import org.apache.ozone.test.GenericTestUtils;
import static org.apache.hadoop.ozone.OzoneConsts.GB;
import static org.apache.hadoop.ozone.OzoneConsts.MB;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -143,7 +146,8 @@ public class TestBlockManager {
scmMetadataStore.getPipelineTable(),
eventQueue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
index acb55ebe0f..433848783c 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.container;
import java.io.File;
import java.io.IOException;
+import java.time.ZoneOffset;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileUtil;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl;
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.common.MonotonicClock;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.ozone.test.GenericTestUtils;
@@ -51,6 +53,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CLOSE_CONTAINER;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND;
+
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -101,7 +104,8 @@ public class TestCloseContainerEventHandler {
scmMetadataStore.getPipelineTable(),
eventQueue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineScrubber.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineScrubber.java
index 8efa4a38ca..3b094c6053 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineScrubber.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestBackgroundPipelineScrubber.java
@@ -21,11 +21,14 @@ import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager.SafeModeStatus;
+import org.apache.ozone.test.TestClock;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -43,19 +46,20 @@ public class TestBackgroundPipelineScrubber {
private SCMContext scmContext;
private PipelineManager pipelineManager;
private OzoneConfiguration conf;
+ private TestClock testClock;
@Before
public void setup() throws IOException {
+ testClock = new TestClock(Instant.now(), ZoneOffset.UTC);
this.scmContext = SCMContext.emptyContext();
this.pipelineManager = mock(PipelineManager.class);
doNothing().when(pipelineManager).scrubPipelines();
- // no initial delay after exit safe mode
this.conf = new OzoneConfiguration();
- conf.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "0ms");
+ conf.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "50s");
this.scrubber = new BackgroundPipelineScrubber(pipelineManager, conf,
- scmContext);
+ scmContext, testClock);
}
@After
@@ -77,6 +81,10 @@ public class TestBackgroundPipelineScrubber {
// out of safe mode, PAUSING -> RUNNING
scrubber.notifyStatusChanged();
+ // Still cannot run, as the safemode delay has not passed.
+ assertFalse(scrubber.shouldRun());
+
+ testClock.fastForward(60000);
assertTrue(scrubber.shouldRun());
// go into safe mode, RUNNING -> PAUSING
@@ -91,6 +99,8 @@ public class TestBackgroundPipelineScrubber {
// kick a run
synchronized (scrubber) {
scrubber.notifyStatusChanged();
+ assertFalse(scrubber.shouldRun());
+ testClock.fastForward(60000);
assertTrue(scrubber.shouldRun());
scrubber.runImmediately();
}
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
index 64ef02eb37..886d565f79 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineManagerImpl.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.TestClock;
import org.apache.ratis.protocol.exceptions.NotLeaderException;
import org.junit.After;
import org.junit.Assert;
@@ -59,6 +60,8 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -88,9 +91,11 @@ public class TestPipelineManagerImpl {
private SCMContext scmContext;
private SCMServiceManager serviceManager;
private StorageContainerManager scm;
+ private TestClock testClock;
@Before
public void init() throws Exception {
+ testClock = new TestClock(Instant.now(), ZoneOffset.UTC);
conf = SCMTestUtils.getConf();
testDir = GenericTestUtils.getTestDir(
TestPipelineManagerImpl.class.getSimpleName() + UUID.randomUUID());
@@ -128,7 +133,8 @@ public class TestPipelineManagerImpl {
SCMDBDefinition.PIPELINES.getTable(dbStore),
new EventQueue(),
scmContext,
- serviceManager);
+ serviceManager,
+ testClock);
}
private PipelineManagerImpl createPipelineManager(
@@ -139,7 +145,8 @@ public class TestPipelineManagerImpl {
SCMDBDefinition.PIPELINES.getTable(dbStore),
new EventQueue(),
SCMContext.emptyContext(),
- serviceManager);
+ serviceManager,
+ new TestClock(Instant.now(), ZoneOffset.UTC));
}
@Test
@@ -536,10 +543,9 @@ public class TestPipelineManagerImpl {
@Test
public void testScrubPipelines() throws Exception {
- // No timeout for pipeline scrubber.
+ // Allocated pipelines should not be scrubbed for 50 seconds.
conf.setTimeDuration(
- OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, -1,
- TimeUnit.MILLISECONDS);
+ OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, 50, TimeUnit.SECONDS);
PipelineManagerImpl pipelineManager = createPipelineManager(true);
pipelineManager.setScmContext(scmContext);
@@ -568,29 +574,41 @@ public class TestPipelineManagerImpl {
.getInstance(ReplicationFactor.THREE),
Pipeline.PipelineState.CLOSED).contains(closedPipeline));
+ // Set the clock to "now". All pipelines were created before this.
+ testClock.set(Instant.now());
+
pipelineManager.scrubPipelines();
- // The allocatedPipeline should be scrubbed.
- Assert.assertFalse(pipelineManager
+ // The allocatedPipeline should not be scrubbed as the interval has not
+ // yet passed.
+ Assert.assertTrue(pipelineManager
.getPipelines(RatisReplicationConfig
.getInstance(ReplicationFactor.THREE),
Pipeline.PipelineState.ALLOCATED).contains(allocatedPipeline));
- // The closedPipeline should be scrubbed.
+ // The closedPipeline should be scrubbed, as they are scrubbed immediately
Assert.assertFalse(pipelineManager
.getPipelines(RatisReplicationConfig
.getInstance(ReplicationFactor.THREE),
Pipeline.PipelineState.CLOSED).contains(closedPipeline));
+ testClock.fastForward((60000));
+
+ pipelineManager.scrubPipelines();
+
+ // The allocatedPipeline should now be scrubbed as the interval has passed
+ Assert.assertFalse(pipelineManager
+ .getPipelines(RatisReplicationConfig
+ .getInstance(ReplicationFactor.THREE),
+ Pipeline.PipelineState.ALLOCATED).contains(allocatedPipeline));
+
pipelineManager.close();
}
@Test
public void testScrubPipelinesShouldFailOnFollower() throws Exception {
- // No timeout for pipeline scrubber.
conf.setTimeDuration(
- OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, -1,
- TimeUnit.MILLISECONDS);
+ OZONE_SCM_PIPELINE_ALLOCATED_TIMEOUT, 10, TimeUnit.SECONDS);
PipelineManagerImpl pipelineManager = createPipelineManager(true);
pipelineManager.setScmContext(scmContext);
@@ -611,6 +629,7 @@ public class TestPipelineManagerImpl {
assert pipelineManager.getScmhaManager() instanceof SCMHAManagerStub;
((SCMHAManagerStub) pipelineManager.getScmhaManager()).setIsLeader(false);
+ testClock.fastForward(20000);
try {
pipelineManager.scrubPipelines();
} catch (NotLeaderException ex) {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
index c6a1021f41..36d396f665 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.scm.safemode;
import java.io.File;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineProvider;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerImpl;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.common.MonotonicClock;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.Assert;
@@ -85,7 +87,8 @@ public class TestHealthyPipelineSafeModeRule {
scmMetadataStore.getPipelineTable(),
eventQueue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
pipelineManager.getStateManager(), config);
@@ -139,7 +142,8 @@ public class TestHealthyPipelineSafeModeRule {
scmMetadataStore.getPipelineTable(),
eventQueue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
@@ -237,7 +241,8 @@ public class TestHealthyPipelineSafeModeRule {
scmMetadataStore.getPipelineTable(),
eventQueue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
index b65fdcab5f..152a4ea42f 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.hdds.scm.safemode;
+import java.time.Instant;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -49,6 +51,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.TestClock;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -97,7 +100,8 @@ public class TestOneReplicaPipelineSafeModeRule {
scmMetadataStore.getPipelineTable(),
eventQueue,
scmContext,
- serviceManager);
+ serviceManager,
+ new TestClock(Instant.now(), ZoneOffset.UTC));
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(mockNodeManager,
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
index 823c2ddf9f..c9197ed6d7 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.safemode;
import java.io.File;
import java.io.IOException;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -53,6 +54,7 @@ import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.common.MonotonicClock;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.After;
@@ -264,7 +266,8 @@ public class TestSCMSafeModeManager {
scmMetadataStore.getPipelineTable(),
queue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
scmSafeModeManager = new SCMSafeModeManager(
conf, containers, null, pipelineManager, queue, serviceManager,
scmContext);
@@ -290,7 +293,8 @@ public class TestSCMSafeModeManager {
scmMetadataStore.getPipelineTable(),
queue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
scmSafeModeManager = new SCMSafeModeManager(
conf, containers, null, pipelineManager, queue, serviceManager,
scmContext);
@@ -315,7 +319,8 @@ public class TestSCMSafeModeManager {
scmMetadataStore.getPipelineTable(),
queue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
scmSafeModeManager = new SCMSafeModeManager(
conf, containers, null, pipelineManager, queue, serviceManager,
scmContext);
@@ -347,7 +352,8 @@ public class TestSCMSafeModeManager {
scmMetadataStore.getPipelineTable(),
queue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(mockNodeManager,
pipelineManager.getStateManager(), config);
@@ -602,7 +608,8 @@ public class TestSCMSafeModeManager {
scmMetadataStore.getPipelineTable(),
queue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
@@ -667,7 +674,8 @@ public class TestSCMSafeModeManager {
scmMetadataStore.getPipelineTable(),
queue,
scmContext,
- serviceManager);
+ serviceManager,
+ new MonotonicClock(ZoneOffset.UTC));
PipelineProvider mockRatisProvider =
new MockRatisPipelineProvider(nodeManager,
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java
index 63f2904e23..93dcae08f8 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconPipelineManager.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.recon.scm;
import java.io.IOException;
+import java.time.ZoneOffset;
import java.util.List;
import java.util.stream.Collectors;
@@ -39,6 +40,7 @@ import org.apache.hadoop.ozone.ClientVersion;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.ozone.common.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,7 +62,8 @@ public final class ReconPipelineManager extends PipelineManagerImpl {
EventPublisher eventPublisher,
SCMContext scmContext) {
super(conf, scmhaManager, nodeManager, pipelineStateManager,
- pipelineFactory, eventPublisher, scmContext);
+ pipelineFactory, eventPublisher, scmContext,
+ new MonotonicClock(ZoneOffset.UTC));
}
public static ReconPipelineManager newReconPipelineManager(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org