You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2022/07/15 14:31:20 UTC
[ozone] branch master updated: HDDS-6987. EC: Implement RECOVERING Container Scrubber (#3591)
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a66e43e283 HDDS-6987. EC: Implement RECOVERING Container Scrubber (#3591)
a66e43e283 is described below
commit a66e43e2838f087229cb4b0fac1e1dc969ab80dd
Author: Jie Yao <ja...@tencent.com>
AuthorDate: Fri Jul 15 22:31:15 2022 +0800
HDDS-6987. EC: Implement RECOVERING Container Scrubber (#3591)
---
.../hadoop/hdds/utils/BackgroundService.java | 9 +
.../apache/hadoop/hdds/utils/BackgroundTask.java | 4 +-
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +++
.../ozone/container/common/impl/ContainerSet.java | 80 ++++++++-
.../common/statemachine/DatanodeConfiguration.java | 17 ++
.../ozone/container/keyvalue/KeyValueHandler.java | 10 +-
.../StaleRecoveringContainerScrubbingService.java | 93 +++++++++++
.../ozone/container/ozoneimpl/OzoneContainer.java | 81 ++++++---
.../container/common/TestBlockDeletingService.java | 42 ++---
.../TestSchemaOneBackwardsCompatibility.java | 2 +-
.../TestSchemaTwoBackwardsCompatibility.java | 2 +-
...stStaleRecoveringContainerScrubbingService.java | 181 +++++++++++++++++++++
.../impl/TestContainerDeletionChoosingPolicy.java | 4 +-
.../common/impl/TestContainerPersistence.java | 2 +-
.../container/common/impl/TestContainerSet.java | 8 +-
.../container/common/impl/TestHddsDispatcher.java | 4 +-
.../TestCloseContainerCommandHandler.java | 2 +-
.../container/keyvalue/TestKeyValueHandler.java | 4 +-
.../container/ozoneimpl/TestContainerReader.java | 4 +-
.../ReplicationSupervisorScheduling.java | 2 +-
.../replication/TestReplicationSupervisor.java | 2 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +
.../transport/server/ratis/TestCSMMetrics.java | 2 +-
.../container/metrics/TestContainerMetrics.java | 2 +-
.../container/server/TestContainerServer.java | 4 +-
.../server/TestSecureContainerServer.java | 4 +-
.../ozone/debug/container/ContainerCommands.java | 4 +-
.../ozone/freon/ClosedContainerReplicator.java | 33 ++--
28 files changed, 533 insertions(+), 96 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java
index 36c1a53709..aa2377075c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundService.java
@@ -77,6 +77,15 @@ public abstract class BackgroundService {
return threadGroup.activeCount();
}
+ @VisibleForTesting
+ public void runPeriodicalTaskNow() throws Exception {
+ BackgroundTaskQueue tasks = getTasks();
+ while (tasks.size() > 0) {
+ tasks.poll().call();
+ }
+ }
+
+
// start service
public void start() {
exec.scheduleWithFixedDelay(service, 0, interval, unit);
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTask.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTask.java
index e985671f04..c97f2e2ced 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTask.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/BackgroundTask.java
@@ -27,5 +27,7 @@ public interface BackgroundTask extends Callable<BackgroundTaskResult> {
@Override
BackgroundTaskResult call() throws Exception;
- int getPriority();
+ default int getPriority() {
+ return 0;
+ }
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index a71d87495b..9894298c41 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -187,6 +187,30 @@ public final class OzoneConfigKeys {
public static final int OZONE_BLOCK_DELETING_SERVICE_WORKERS_DEFAULT
= 10;
+ /**
+ * Configuration properties for Ozone Recovering Container Scrubbing Service.
+ */
+ public static final String
+ OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT =
+ "ozone.recovering.container.scrubbing.service.timeout";
+
+ // 300s for default
+ public static final String
+ OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT_DEFAULT = "300s";
+
+ public static final String
+ OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_WORKERS =
+ "ozone.recovering.container.scrubbing.service.workers";
+ public static final int
+ OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_WORKERS_DEFAULT = 10;
+
+ public static final String
+ OZONE_RECOVERING_CONTAINER_TIMEOUT =
+ "ozone.recovering.container.timeout";
+ public static final String
+ OZONE_RECOVERING_CONTAINER_TIMEOUT_DEFAULT = "20m";
+
+
public static final String OZONE_KEY_PREALLOCATION_BLOCKS_MAX =
"ozone.key.preallocation.max.blocks";
public static final int OZONE_KEY_PREALLOCATION_BLOCKS_MAX_DEFAULT
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
index 6aaf6fc457..3e81333d34 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
@@ -22,26 +22,29 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers
- .StorageContainerException;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.common.MonotonicClock;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.time.Clock;
+import java.time.ZoneOffset;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Iterator;
-import java.util.Set;
import java.util.List;
-import java.util.Collections;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.ConcurrentSkipListSet;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING;
+
/**
* Class that manages Containers created on the datanode.
@@ -54,6 +57,30 @@ public class ContainerSet {
ConcurrentSkipListMap<>();
private final ConcurrentSkipListSet<Long> missingContainerSet =
new ConcurrentSkipListSet<>();
+ private final ConcurrentSkipListMap<Long, Long> recoveringContainerMap =
+ new ConcurrentSkipListMap<>();
+ private Clock clock;
+ private long recoveringTimeout;
+
+ public ContainerSet(long recoveringTimeout) {
+ this.clock = new MonotonicClock(ZoneOffset.UTC);
+ this.recoveringTimeout = recoveringTimeout;
+ }
+
+ public long getCurrentTime() {
+ return clock.millis();
+ }
+
+ @VisibleForTesting
+ public void setClock(Clock clock) {
+ this.clock = clock;
+ }
+
+ @VisibleForTesting
+ public void setRecoveringTimeout(long recoveringTimeout) {
+ this.recoveringTimeout = recoveringTimeout;
+ }
+
/**
* Add Container to container map.
* @param container container to be added
@@ -72,6 +99,10 @@ public class ContainerSet {
}
// wish we could have done this from ContainerData.setState
container.getContainerData().commitSpace();
+ if (container.getContainerData().getState() == RECOVERING) {
+ recoveringContainerMap.put(
+ clock.millis() + recoveringTimeout, containerId);
+ }
return true;
} else {
LOG.warn("Container already exists with container Id {}", containerId);
@@ -113,6 +144,33 @@ public class ContainerSet {
}
}
+ /**
+ * Removes the Recovering Container matching with specified containerId.
+ * @param containerId ID of the container to remove.
+ * @return true If container is removed from containerMap returns true,
+ * otherwise false.
+ */
+ public boolean removeRecoveringContainer(long containerId) {
+ Preconditions.checkState(containerId >= 0,
+ "Container Id cannot be negative.");
+ //it might take a little long time to iterate all the entries
+ // in recoveringContainerMap, but it seems ok here since:
+ // 1 In the vast majority of cases，there will not be too
+ // many recovering containers.
+ // 2 closing container is not a sort of urgent action
+ //
+ // we can revisit here if any performance problem happens
+ Iterator<Map.Entry<Long, Long>> it = getRecoveringContainerIterator();
+ while (it.hasNext()) {
+ Map.Entry<Long, Long> entry = it.next();
+ if (entry.getValue() == containerId) {
+ it.remove();
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Return number of containers in container map.
* @return container count
@@ -147,6 +205,15 @@ public class ContainerSet {
return containerMap.values().iterator();
}
+ /**
+ * Return an container Iterator over
+ * {@link ContainerSet#recoveringContainerMap}.
+ * @return {@literal Iterator<Container<?>>}
+ */
+ public Iterator<Map.Entry<Long, Long>> getRecoveringContainerIterator() {
+ return recoveringContainerMap.entrySet().iterator();
+ }
+
/**
* Return an iterator of containers associated with the specified volume.
* The iterator is sorted by last data scan timestamp in increasing order.
@@ -302,6 +369,5 @@ public class ContainerSet {
}
}
});
-
}
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
index 3a11c53289..2ce12cae70 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java
@@ -145,10 +145,27 @@ public class DatanodeConfiguration {
)
private long blockDeletionInterval = Duration.ofSeconds(60).toMillis();
+ @Config(key = "recovering.container.scrubbing.service.interval",
+ defaultValue = "1m",
+ type = ConfigType.TIME,
+ tags = { ConfigTag.SCM, ConfigTag.DELETION },
+ description =
+ "Time interval of the stale recovering container scrubbing " +
+ "service. The recovering container scrubbing service runs " +
+ "on Datanode periodically and deletes stale recovering " +
+ "container Unit could be defined with postfix (ns,ms,s,m,h,d)."
+ )
+ private long recoveringContainerScrubInterval =
+ Duration.ofMinutes(10).toMillis();
+
public Duration getBlockDeletionInterval() {
return Duration.ofMillis(blockDeletionInterval);
}
+ public Duration getRecoveringContainerScrubInterval() {
+ return Duration.ofMillis(recoveringContainerScrubInterval);
+ }
+
public void setBlockDeletionInterval(Duration duration) {
this.blockDeletionInterval = duration.toMillis();
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 7d88aefeed..b15e48935a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -105,6 +105,8 @@ import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuil
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.putBlockResponseSuccess;
import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest;
import static org.apache.hadoop.hdds.scm.utils.ClientCommandsUtils.getReadChunkVersion;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
+ .ContainerDataProto.State.RECOVERING;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.slf4j.Logger;
@@ -989,8 +991,14 @@ public class KeyValueHandler extends Handler {
throws IOException {
container.writeLock();
try {
+ ContainerProtos.ContainerDataProto.State state =
+ container.getContainerState();
// Move the container to CLOSING state only if it's OPEN/RECOVERING
- if (HddsUtils.isOpenToWriteState(container.getContainerState())) {
+ if (HddsUtils.isOpenToWriteState(state)) {
+ if (state == RECOVERING) {
+ containerSet.removeRecoveringContainer(
+ container.getContainerData().getContainerID());
+ }
container.markContainerForClose();
sendICR(container);
}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
new file mode 100644
index 0000000000..b73bc5a435
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/StaleRecoveringContainerScrubbingService.java
@@ -0,0 +1,93 @@
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.statemachine.background;
+
+import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.BackgroundTask;
+import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
+import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A per-datanode container stale recovering container scrubbing service
+ * takes in charge of deleting stale recovering containers.
+ */
+
+public class StaleRecoveringContainerScrubbingService
+ extends BackgroundService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StaleRecoveringContainerScrubbingService.class);
+
+ private final ContainerSet containerSet;
+
+ public StaleRecoveringContainerScrubbingService(
+ long interval, TimeUnit unit, int threadPoolSize,
+ long serviceTimeout, ContainerSet containerSet) {
+ super("StaleRecoveringContainerScrubbingService",
+ interval, unit, threadPoolSize, serviceTimeout);
+ this.containerSet = containerSet;
+
+ }
+
+ @Override
+ public BackgroundTaskQueue getTasks() {
+ BackgroundTaskQueue backgroundTaskQueue =
+ new BackgroundTaskQueue();
+ long currentTime = containerSet.getCurrentTime();
+ Iterator<Map.Entry<Long, Long>> it =
+ containerSet.getRecoveringContainerIterator();
+ while (it.hasNext()) {
+ Map.Entry<Long, Long> entry = it.next();
+ if (currentTime >= entry.getKey()) {
+ backgroundTaskQueue.add(new RecoveringContainerScrubbingTask(
+ containerSet, entry.getValue()));
+ it.remove();
+ } else {
+ break;
+ }
+ }
+ return backgroundTaskQueue;
+ }
+
+ static class RecoveringContainerScrubbingTask implements BackgroundTask {
+ private final ContainerSet containerSet;
+ private final long containerID;
+
+ RecoveringContainerScrubbingTask(
+ ContainerSet containerSet, long containerID) {
+ this.containerSet = containerSet;
+ this.containerID = containerID;
+ }
+
+ @Override
+ public BackgroundTaskResult call() throws Exception {
+ containerSet.getContainer(containerID).delete();
+ containerSet.removeContainer(containerID);
+ LOG.info("Delete stale recovering container {}", containerID);
+ return new BackgroundTaskResult.EmptyTaskResult();
+ }
+ }
+}
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 1db2961ba9..d2a9480434 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -18,16 +18,8 @@
package org.apache.hadoop.ozone.container.ozoneimpl;
-import java.io.IOException;
-import java.security.cert.X509Certificate;
-import java.time.Duration;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name;
@@ -62,24 +54,38 @@ import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume.VolumeType;
import org.apache.hadoop.ozone.container.common.volume.StorageVolumeChecker;
import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService;
+import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService;
import org.apache.hadoop.ozone.container.replication.ReplicationServer;
import org.apache.hadoop.ozone.container.replication.ReplicationServer.ReplicationConfig;
import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures.SchemaV3;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
+import org.apache.hadoop.util.Timer;
+import org.apache.ratis.grpc.GrpcTlsConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Maps;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_WORKERS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_WORKERS_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_WORKERS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_WORKERS_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_TIMEOUT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone.container.ozoneimpl.ContainerScrubberConfiguration.VOLUME_BYTES_PER_SECOND_KEY;
-import org.apache.hadoop.util.Timer;
-import org.apache.ratis.grpc.GrpcTlsConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* Ozone main class sets up the network servers and initializes the container
* layer.
@@ -103,6 +109,8 @@ public class OzoneContainer {
private ContainerMetadataScanner metadataScanner;
private List<ContainerDataScanner> dataScanners;
private final BlockDeletingService blockDeletingService;
+ private final StaleRecoveringContainerScrubbingService
+ recoveringContainerScrubbingService;
private final GrpcTlsConfig tlsClientConfig;
private final AtomicReference<InitializingStatus> initializingStatus;
private final ReplicationServer replicationServer;
@@ -143,7 +151,11 @@ public class OzoneContainer {
HddsVolumeUtil.loadAllHddsVolumeDbStore(volumeSet, dbVolumeSet, LOG);
}
- containerSet = new ContainerSet();
+ long recoveringContainerTimeout = config.getTimeDuration(
+ OZONE_RECOVERING_CONTAINER_TIMEOUT,
+ OZONE_RECOVERING_CONTAINER_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
+
+ containerSet = new ContainerSet(recoveringContainerTimeout);
metadataScanner = null;
buildContainerSet();
@@ -194,20 +206,41 @@ public class OzoneContainer {
readChannel = new XceiverServerGrpc(
datanodeDetails, config, hddsDispatcher, certClient);
- Duration svcInterval = conf.getObject(
+ Duration blockDeletingSvcInterval = conf.getObject(
DatanodeConfiguration.class).getBlockDeletionInterval();
- long serviceTimeout = config
+ long blockDeletingServiceTimeout = config
.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
- int serviceWorkerSize = config
+ int blockDeletingServiceWorkerSize = config
.getInt(OZONE_BLOCK_DELETING_SERVICE_WORKERS,
OZONE_BLOCK_DELETING_SERVICE_WORKERS_DEFAULT);
blockDeletingService =
- new BlockDeletingService(this, svcInterval.toMillis(), serviceTimeout,
- TimeUnit.MILLISECONDS, serviceWorkerSize, config);
+ new BlockDeletingService(this, blockDeletingSvcInterval.toMillis(),
+ blockDeletingServiceTimeout, TimeUnit.MILLISECONDS,
+ blockDeletingServiceWorkerSize, config);
+
+ Duration recoveringContainerScrubbingSvcInterval = conf.getObject(
+ DatanodeConfiguration.class).getRecoveringContainerScrubInterval();
+
+ long recoveringContainerScrubbingServiceTimeout = config
+ .getTimeDuration(OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT,
+ OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
+ int recoveringContainerScrubbingServiceWorkerSize = config
+ .getInt(OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_WORKERS,
+ OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_WORKERS_DEFAULT);
+
+ recoveringContainerScrubbingService =
+ new StaleRecoveringContainerScrubbingService(
+ recoveringContainerScrubbingSvcInterval.toMillis(),
+ TimeUnit.MILLISECONDS,
+ recoveringContainerScrubbingServiceWorkerSize,
+ recoveringContainerScrubbingServiceTimeout,
+ containerSet);
if (certClient != null && secConf.isGrpcTlsEnabled()) {
List<X509Certificate> x509Certificates =
@@ -352,6 +385,7 @@ public class OzoneContainer {
hddsDispatcher.init();
hddsDispatcher.setClusterId(clusterId);
blockDeletingService.start();
+ recoveringContainerScrubbingService.start();
// mark OzoneContainer as INITIALIZED.
initializingStatus.set(InitializingStatus.INITIALIZED);
@@ -376,6 +410,7 @@ public class OzoneContainer {
dbVolumeSet.shutdown();
}
blockDeletingService.shutdown();
+ recoveringContainerScrubbingService.shutdown();
ContainerMetrics.remove();
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 8888151511..9539d5bf03 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -18,16 +18,6 @@
package org.apache.hadoop.ozone.container.common;
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.UUID;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
@@ -48,13 +38,14 @@ import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.BlockDeletingServiceMetrics;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
-import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
@@ -75,10 +66,6 @@ import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.container.testutils.BlockDeletingServiceTestImpl;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
-
-import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
-
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -86,6 +73,18 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
@@ -98,7 +97,6 @@ import static org.apache.hadoop.ozone.container.common.states.endpoint.VersionEn
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import static java.nio.charset.StandardCharsets.UTF_8;
/**
* Tests to test block deleting service.
@@ -413,7 +411,7 @@ public class TestBlockDeletingService {
dnConf.setBlockDeletionLimit(2);
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
conf.setFromObject(dnConf);
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
createToDeleteBlocks(containerSet, 1, 3, 1);
ContainerMetrics metrics = ContainerMetrics.create(conf);
KeyValueHandler keyValueHandler =
@@ -501,7 +499,8 @@ public class TestBlockDeletingService {
TimeUnit.MILLISECONDS);
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 10);
- ContainerSet containerSet = new ContainerSet();
+
+ ContainerSet containerSet = new ContainerSet(1000);
// Create 1 container with 100 blocks
createToDeleteBlocks(containerSet, 1, 100, 1);
ContainerMetrics metrics = ContainerMetrics.create(conf);
@@ -529,7 +528,8 @@ public class TestBlockDeletingService {
dnConf.setBlockDeletionLimit(3);
blockLimitPerInterval = dnConf.getBlockDeletionLimit();
conf.setFromObject(dnConf);
- ContainerSet containerSet = new ContainerSet();
+
+ ContainerSet containerSet = new ContainerSet(1000);
createToDeleteBlocks(containerSet, 1, 3, 1);
ContainerMetrics metrics = ContainerMetrics.create(conf);
KeyValueHandler keyValueHandler =
@@ -629,7 +629,7 @@ public class TestBlockDeletingService {
dnConf.setBlockDeletionLimit(1);
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
conf.setFromObject(dnConf);
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
int containerCount = 2;
int chunksPerBlock = 10;
@@ -700,7 +700,7 @@ public class TestBlockDeletingService {
dnConf.setBlockDeletionLimit(10);
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
conf.setFromObject(dnConf);
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
ContainerMetrics metrics = ContainerMetrics.create(conf);
KeyValueHandler keyValueHandler =
new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet,
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
index 6e2e8654f3..1323a98b1e 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
@@ -552,7 +552,7 @@ public class TestSchemaOneBackwardsCompatibility {
}
private ContainerSet makeContainerSet() throws Exception {
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
KeyValueContainer container = new KeyValueContainer(newKvData(), conf);
containerSet.addContainer(container);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java
index 47fd151403..d2741dc9cd 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java
@@ -146,7 +146,7 @@ public class TestSchemaTwoBackwardsCompatibility {
blockManager = new BlockManagerImpl(conf);
chunkManager = new FilePerBlockStrategy(true, blockManager, volumeSet);
- containerSet = new ContainerSet();
+ containerSet = new ContainerSet(1000);
keyValueHandler = new KeyValueHandler(conf, datanodeUuid,
containerSet, volumeSet, ContainerMetrics.create(conf), c -> { });
ozoneContainer = mock(OzoneContainer.class);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
new file mode 100644
index 0000000000..fea6c776db
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.common;
+
+import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
+import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
+import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
+import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.StaleRecoveringContainerScrubbingService;
+import org.apache.ozone.test.TestClock;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.Iterator;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests to stale recovering container scrubbing service.
+ */
+@RunWith(Parameterized.class)
+public class TestStaleRecoveringContainerScrubbingService {
+
+ @Rule
+ public final TemporaryFolder tempDir = new TemporaryFolder();
+ private String datanodeUuid;
+ private OzoneConfiguration conf;
+ private HddsVolume hddsVolume;
+
+ private final ContainerLayoutVersion layout;
+ private final String schemaVersion;
+ private String clusterID;
+ private int containerIdNum = 0;
+ private MutableVolumeSet volumeSet;
+ private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy;
+ private final TestClock testClock =
+ new TestClock(Instant.now(), ZoneOffset.UTC);
+
+ public TestStaleRecoveringContainerScrubbingService(
+ ContainerTestVersionInfo versionInfo) {
+ this.layout = versionInfo.getLayout();
+ this.schemaVersion = versionInfo.getSchemaVersion();
+ conf = new OzoneConfiguration();
+ ContainerTestVersionInfo.setTestSchemaVersion(schemaVersion, conf);
+ }
+
+ @Parameterized.Parameters
+ public static Iterable<Object[]> parameters() {
+ return ContainerTestVersionInfo.versionParameters();
+ }
+
+ @Before
+ public void init() throws IOException {
+ File volumeDir = tempDir.newFolder();
+ conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, volumeDir.getAbsolutePath());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, volumeDir.getAbsolutePath());
+ datanodeUuid = UUID.randomUUID().toString();
+ clusterID = UUID.randomUUID().toString();
+ hddsVolume = new HddsVolume.Builder(volumeDir.getAbsolutePath())
+ .conf(conf).datanodeUuid(datanodeUuid).clusterID(clusterID).build();
+ hddsVolume.format(clusterID);
+ hddsVolume.createWorkingDir(clusterID, null);
+ volumeSet = mock(MutableVolumeSet.class);
+
+ volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class);
+ Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong()))
+ .thenReturn(hddsVolume);
+ }
+
+ @After
+ public void cleanup() throws IOException {
+ BlockUtils.shutdownCache(conf);
+ }
+
+ /**
+ * A helper method to create a number of containers of given state.
+ */
+ private void createTestContainers(
+ ContainerSet containerSet, int num,
+ ContainerProtos.ContainerDataProto.State state)
+ throws StorageContainerException {
+ int end = containerIdNum + num;
+ for (; containerIdNum < end; containerIdNum++) {
+ testClock.fastForward(10L);
+ KeyValueContainerData recoveringContainerData = new KeyValueContainerData(
+ containerIdNum, layout, (long) StorageUnit.GB.toBytes(5),
+ UUID.randomUUID().toString(), datanodeUuid);
+ //create a container with recovering state
+ recoveringContainerData.setState(state);
+
+ KeyValueContainer recoveringKeyValueContainer =
+ new KeyValueContainer(recoveringContainerData,
+ conf);
+ recoveringKeyValueContainer.create(
+ volumeSet, volumeChoosingPolicy, clusterID);
+ containerSet.addContainer(recoveringKeyValueContainer);
+ }
+ }
+
+ @Test
+ public void testScrubbingStaleRecoveringContainers()
+ throws Exception {
+ ContainerSet containerSet = new ContainerSet(10);
+ containerSet.setClock(testClock);
+ StaleRecoveringContainerScrubbingService srcss =
+ new StaleRecoveringContainerScrubbingService(
+ 50, TimeUnit.MILLISECONDS, 10,
+ Duration.ofSeconds(300).toMillis(),
+ containerSet);
+ testClock.fastForward(1000L);
+ createTestContainers(containerSet, 5, CLOSED);
+ testClock.fastForward(1000L);
+ srcss.runPeriodicalTaskNow();
+ //closed container should not be scrubbed
+ Assert.assertTrue(containerSet.containerCount() == 5);
+
+ createTestContainers(containerSet, 5, RECOVERING);
+ testClock.fastForward(1000L);
+ srcss.runPeriodicalTaskNow();
+ //recovering container should be scrubbed since recovering timeout
+ Assert.assertTrue(containerSet.containerCount() == 5);
+ Iterator<Container<?>> it = containerSet.getContainerIterator();
+ while (it.hasNext()) {
+ Container<?> entry = it.next();
+ Assert.assertTrue(entry.getContainerState().equals(CLOSED));
+ }
+
+ //increase recovering timeout
+ containerSet.setRecoveringTimeout(2000L);
+ createTestContainers(containerSet, 5, RECOVERING);
+ testClock.fastForward(1000L);
+ srcss.runPeriodicalTaskNow();
+ //recovering container should not be scrubbed
+ Assert.assertTrue(containerSet.containerCount() == 10);
+ }
+}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
index 94f2e3ebae..cb86bf38e7 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
@@ -94,7 +94,7 @@ public class TestContainerDeletionChoosingPolicy {
RandomContainerDeletionChoosingPolicy.class.getName());
List<StorageLocation> pathLists = new LinkedList<>();
pathLists.add(StorageLocation.parse(containerDir.getAbsolutePath()));
- containerSet = new ContainerSet();
+ containerSet = new ContainerSet(1000);
int numContainers = 10;
for (int i = 0; i < numContainers; i++) {
@@ -156,7 +156,7 @@ public class TestContainerDeletionChoosingPolicy {
TopNOrderedContainerDeletionChoosingPolicy.class.getName());
List<StorageLocation> pathLists = new LinkedList<>();
pathLists.add(StorageLocation.parse(containerDir.getAbsolutePath()));
- containerSet = new ContainerSet();
+ containerSet = new ContainerSet(1000);
int numContainers = 10;
Random random = new Random();
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
index 09dd32b1d7..c1a523719e 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
@@ -147,7 +147,7 @@ public class TestContainerPersistence {
@Before
public void setupPaths() throws IOException {
- containerSet = new ContainerSet();
+ containerSet = new ContainerSet(1000);
volumeSet = new MutableVolumeSet(DATANODE_UUID, conf, null,
StorageVolume.VolumeType.DATA_VOLUME, null);
createDbInstancesForTestIfNeeded(volumeSet, SCM_ID, SCM_ID, conf);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
index d51d78e4ad..929f50da64 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
@@ -74,7 +74,7 @@ public class TestContainerSet {
@Test
public void testAddGetRemoveContainer() throws StorageContainerException {
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
long containerId = 100L;
ContainerProtos.ContainerDataProto.State state = ContainerProtos
.ContainerDataProto.State.CLOSED;
@@ -166,7 +166,7 @@ public class TestContainerSet {
HddsVolume vol2 = Mockito.mock(HddsVolume.class);
Mockito.when(vol2.getStorageID()).thenReturn("uuid-2");
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
for (int i = 0; i < 10; i++) {
KeyValueContainerData kvData = new KeyValueContainerData(i,
layout,
@@ -207,7 +207,7 @@ public class TestContainerSet {
HddsVolume vol = Mockito.mock(HddsVolume.class);
Mockito.when(vol.getStorageID()).thenReturn("uuid-1");
Random random = new Random();
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
int containerCount = 50;
for (int i = 0; i < containerCount; i++) {
KeyValueContainerData kvData = new KeyValueContainerData(i,
@@ -301,7 +301,7 @@ public class TestContainerSet {
}
private ContainerSet createContainerSet() throws StorageContainerException {
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
for (int i = FIRST_ID; i < FIRST_ID + 10; i++) {
KeyValueContainerData kvData = new KeyValueContainerData(i,
layout,
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index baceb28b63..8a6dedcc03 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -109,7 +109,7 @@ public class TestHddsDispatcher {
try {
UUID scmId = UUID.randomUUID();
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
DatanodeStateMachine stateMachine = Mockito.mock(
DatanodeStateMachine.class);
@@ -302,7 +302,7 @@ public class TestHddsDispatcher {
*/
private HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId,
OzoneConfiguration conf) throws IOException {
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
VolumeSet volumeSet = new MutableVolumeSet(dd.getUuidString(), conf, null,
StorageVolume.VolumeType.DATA_VOLUME, null);
DatanodeStateMachine stateMachine = Mockito.mock(
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
index c30ded0051..f8a9c360c0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
@@ -98,7 +98,7 @@ public class TestCloseContainerCommandHandler {
pipelineID.getId().toString(), null);
container = new KeyValueContainer(data, new OzoneConfiguration());
- containerSet = new ContainerSet();
+ containerSet = new ContainerSet(1000);
containerSet.addContainer(container);
containerHandler = mock(Handler.class);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 7d2d604db5..991ab7a58a 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -260,7 +260,7 @@ public class TestKeyValueHandler {
volumeSet = new MutableVolumeSet(UUID.randomUUID().toString(), conf,
null, StorageVolume.VolumeType.DATA_VOLUME, null);
try {
- ContainerSet cset = new ContainerSet();
+ ContainerSet cset = new ContainerSet(1000);
int[] interval = new int[1];
interval[0] = 2;
ContainerMetrics metrics = new ContainerMetrics(interval);
@@ -350,7 +350,7 @@ public class TestKeyValueHandler {
try {
final long containerID = 1L;
final ConfigurationSource conf = new OzoneConfiguration();
- final ContainerSet containerSet = new ContainerSet();
+ final ContainerSet containerSet = new ContainerSet(1000);
final VolumeSet volumeSet = Mockito.mock(VolumeSet.class);
Mockito.when(volumeSet.getVolumesList())
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
index 4f633b1d4e..a17a5f7f8c 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
@@ -105,7 +105,7 @@ public class TestContainerReader {
File volumeDir = tempDir.newFolder();
volumeSet = Mockito.mock(MutableVolumeSet.class);
- containerSet = new ContainerSet();
+ containerSet = new ContainerSet(1000);
datanodeId = UUID.randomUUID();
hddsVolume = new HddsVolume.Builder(volumeDir
@@ -261,7 +261,7 @@ public class TestContainerReader {
public void testContainerReaderWithLoadException() throws Exception {
MutableVolumeSet volumeSet1;
HddsVolume hddsVolume1;
- ContainerSet containerSet1 = new ContainerSet();
+ ContainerSet containerSet1 = new ContainerSet(1000);
File volumeDir1 = tempDir.newFolder();
RoundRobinVolumeChoosingPolicy volumeChoosingPolicy1;
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorScheduling.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorScheduling.java
index ebdfad6360..486104bd75 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorScheduling.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisorScheduling.java
@@ -67,7 +67,7 @@ public class ReplicationSupervisorScheduling {
destinationLocks.put(i, new Object());
}
- ContainerSet cs = new ContainerSet();
+ ContainerSet cs = new ContainerSet(1000);
ReplicationSupervisor rs = new ReplicationSupervisor(cs,
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
index 8078fc25c8..51f78440bb 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@ -88,7 +88,7 @@ public class TestReplicationSupervisor {
@Before
public void setUp() throws Exception {
- set = new ContainerSet();
+ set = new ContainerSet(1000);
}
@After
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 5fd2eb0ea7..137ae08dad 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -104,6 +104,9 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase {
OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE,
OzoneConfigKeys.OZONE_S3_AUTHINFO_MAX_LIFETIME_KEY,
OzoneConfigKeys.OZONE_CLIENT_REQUIRED_OM_VERSION_MIN_KEY,
+ OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_WORKERS,
+ OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT,
+ OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_TIMEOUT,
ReconConfigKeys.RECON_SCM_CONFIG_PREFIX,
ReconConfigKeys.OZONE_RECON_ADDRESS_KEY,
ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY,
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
index 28227318ed..7467d4a0e8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
@@ -194,7 +194,7 @@ public class TestCSMMetrics {
final ContainerDispatcher dispatcher = new TestContainerDispatcher();
return XceiverServerRatis.newXceiverServerRatis(dn, conf, dispatcher,
- new ContainerController(new ContainerSet(), Maps.newHashMap()),
+ new ContainerController(new ContainerSet(1000), Maps.newHashMap()),
null, null);
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
index fb015bb122..1bc5d81ef0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
@@ -97,7 +97,7 @@ public class TestContainerMetrics {
VolumeSet volumeSet = new MutableVolumeSet(
datanodeDetails.getUuidString(), conf,
null, StorageVolume.VolumeType.DATA_VOLUME, null);
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
DatanodeStateMachine stateMachine = Mockito.mock(
DatanodeStateMachine.class);
StateContext context = Mockito.mock(StateContext.class);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
index 57e291e445..1cafedc32b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
@@ -123,7 +123,7 @@ public class TestContainerServer {
final ContainerDispatcher dispatcher = new TestContainerDispatcher();
return XceiverServerRatis.newXceiverServerRatis(dn, conf, dispatcher,
- new ContainerController(new ContainerSet(), Maps.newHashMap()),
+ new ContainerController(new ContainerSet(1000), Maps.newHashMap()),
caClient, null);
}
@@ -190,7 +190,7 @@ public class TestContainerServer {
pipeline.getFirstNode()
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
VolumeSet volumeSet = mock(MutableVolumeSet.class);
ContainerMetrics metrics = ContainerMetrics.create(conf);
Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
index 361158edb9..3041078fab 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
@@ -172,7 +172,7 @@ public class TestSecureContainerServer {
private static HddsDispatcher createDispatcher(DatanodeDetails dd, UUID scmId,
OzoneConfiguration conf) throws IOException {
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
conf.set(HDDS_DATANODE_DIR_KEY,
Paths.get(TEST_DIR, "dfs", "data", "hdds",
RandomStringUtils.randomAlphabetic(4)).toString());
@@ -221,7 +221,7 @@ public class TestSecureContainerServer {
final ContainerDispatcher dispatcher = createDispatcher(dn,
UUID.randomUUID(), conf);
return XceiverServerRatis.newXceiverServerRatis(dn, conf, dispatcher,
- new ContainerController(new ContainerSet(), Maps.newHashMap()),
+ new ContainerController(new ContainerSet(1000), Maps.newHashMap()),
caClient, null);
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java
index 453370ed07..60f8750867 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/container/ContainerCommands.java
@@ -46,8 +46,8 @@ import org.kohsuke.MetaInfServices;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;
-import picocli.CommandLine.ParentCommand;
import picocli.CommandLine.Model.CommandSpec;
+import picocli.CommandLine.ParentCommand;
import picocli.CommandLine.Spec;
import java.io.File;
@@ -108,7 +108,7 @@ public class ContainerCommands implements Callable<Void>, SubcommandWithParent {
public void loadContainersFromVolumes() throws IOException {
OzoneConfiguration conf = parent.getOzoneConf();
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
ContainerMetrics metrics = ContainerMetrics.create(conf);
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java
index 6797203d81..a7d332e783 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java
@@ -16,20 +16,7 @@
*/
package org.apache.hadoop.ozone.freon;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
+import com.codahale.metrics.Timer;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -53,12 +40,24 @@ import org.apache.hadoop.ozone.container.replication.DownloadAndImportReplicator
import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
import org.apache.hadoop.ozone.container.replication.ReplicationTask;
import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
-
-import com.codahale.metrics.Timer;
import org.jetbrains.annotations.NotNull;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
/**
* Utility to replicated closed container with datanode code.
*/
@@ -171,7 +170,7 @@ public class ClosedContainerReplicator extends BaseFreonGenerator implements
fakeDatanodeUuid = UUID.randomUUID().toString();
}
- ContainerSet containerSet = new ContainerSet();
+ ContainerSet containerSet = new ContainerSet(1000);
ContainerMetrics metrics = ContainerMetrics.create(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org