You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by iv...@apache.org on 2026/05/01 01:33:28 UTC
(ozone) branch master updated: HDDS-10767. Reducing DatanodeDetails in the ContainerLocationCache (#10158)
This is an automated email from the ASF dual-hosted git repository.
ivandika3 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e215b48bfdd HDDS-10767. Reducing DatanodeDetails in the ContainerLocationCache (#10158)
e215b48bfdd is described below
commit e215b48bfdd19e0c4a3ee1b2d4e4a232b6640deb
Author: Ivan Andika <iv...@apache.org>
AuthorDate: Fri May 1 09:33:21 2026 +0800
HDDS-10767. Reducing DatanodeDetails in the ContainerLocationCache (#10158)
Co-authored-by: guohao1 <gu...@360.cn>
---
.../hadoop/hdds/protocol/DatanodeDetails.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 10 ++++
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 5 ++
.../java/org/apache/hadoop/ozone/om/ScmClient.java | 57 ++++++++++++++++++++--
.../org/apache/hadoop/ozone/om/TestScmClient.java | 55 +++++++++++++++++++--
5 files changed, 118 insertions(+), 11 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
index d72e010535d..e75e1a5c5b3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
@@ -758,7 +758,7 @@ public Builder setDatanodeDetails(DatanodeDetails details) {
this.id = details.id;
this.ipAddress = details.getIpAddressAsByteString();
this.hostName = details.getHostNameAsByteString();
- this.networkName = details.getHostNameAsByteString();
+ this.networkName = details.getNetworkNameAsByteString();
this.networkLocation = details.getNetworkLocationAsByteString();
this.level = details.getLevel();
this.ports = details.getPorts();
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index aeda046dcd7..8cbafc61f09 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -4385,6 +4385,16 @@
</description>
</property>
+ <property>
+ <name>ozone.om.container.location.datanode.cache.size</name>
+ <value>10000</value>
+ <tag>OZONE, OM</tag>
+ <description>
+ The size of the datanode details cache used to reduce duplicate datanode objects
+ retained by the container location cache in Ozone Manager.
+ </description>
+ </property>
+
<property>
<name>ozone.om.container.location.cache.ttl</name>
<value>360m</value>
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index b9ccfedb40d..ea17ad9ca85 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -550,6 +550,11 @@ public final class OMConfigKeys {
public static final int OZONE_OM_CONTAINER_LOCATION_CACHE_SIZE_DEFAULT
= 100_000;
+ public static final String OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE
+ = "ozone.om.container.location.datanode.cache.size";
+ public static final int
+ OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE_DEFAULT = 10_000;
+
public static final String OZONE_OM_CONTAINER_LOCATION_CACHE_TTL
= "ozone.om.container.location.cache.ttl";
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java
index 0bdcce56e5c..868a248e909 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ScmClient.java
@@ -21,13 +21,17 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_CACHE_SIZE_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_CACHE_TTL;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_CACHE_TTL_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE_DEFAULT;
+import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
import com.google.common.cache.LoadingCache;
import jakarta.annotation.Nonnull;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -37,7 +41,8 @@
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
@@ -52,21 +57,28 @@ public class ScmClient {
private final StorageContainerLocationProtocol containerClient;
private final LoadingCache<Long, Pipeline> containerLocationCache;
private final CacheMetrics containerCacheMetrics;
+ private final CacheMetrics datanodeDetailsCacheMetrics;
ScmClient(ScmBlockLocationProtocol blockClient,
StorageContainerLocationProtocol containerClient,
OzoneConfiguration configuration) {
this.containerClient = containerClient;
this.blockClient = blockClient;
+ Cache<DatanodeID, DatanodeDetails> datanodeDetailsCache =
+ createDatanodeDetailsCache(configuration);
this.containerLocationCache =
- createContainerLocationCache(configuration, containerClient);
+ createContainerLocationCache(configuration, containerClient,
+ datanodeDetailsCache);
this.containerCacheMetrics = CacheMetrics.create(containerLocationCache,
"ContainerInfo");
+ this.datanodeDetailsCacheMetrics = CacheMetrics.create(
+ datanodeDetailsCache, "DatanodeDetails");
}
static LoadingCache<Long, Pipeline> createContainerLocationCache(
OzoneConfiguration configuration,
- StorageContainerLocationProtocol containerClient) {
+ StorageContainerLocationProtocol containerClient,
+ Cache<DatanodeID, DatanodeDetails> datanodeDetailsCache) {
int maxSize = configuration.getInt(OZONE_OM_CONTAINER_LOCATION_CACHE_SIZE,
OZONE_OM_CONTAINER_LOCATION_CACHE_SIZE_DEFAULT);
TimeUnit unit = OZONE_OM_CONTAINER_LOCATION_CACHE_TTL_DEFAULT.getUnit();
@@ -81,7 +93,9 @@ static LoadingCache<Long, Pipeline> createContainerLocationCache(
@Nonnull
@Override
public Pipeline load(@Nonnull Long key) throws Exception {
- return containerClient.getContainerWithPipeline(key).getPipeline();
+ Pipeline pipeline =
+ containerClient.getContainerWithPipeline(key).getPipeline();
+ return newPipelineWithDNCache(pipeline, datanodeDetailsCache);
}
@Nonnull
@@ -92,12 +106,44 @@ public Map<Long, Pipeline> loadAll(
.stream()
.collect(Collectors.toMap(
x -> x.getContainerInfo().getContainerID(),
- ContainerWithPipeline::getPipeline
+ x -> newPipelineWithDNCache(x.getPipeline(),
+ datanodeDetailsCache)
));
}
});
}
+ static Cache<DatanodeID, DatanodeDetails> createDatanodeDetailsCache(
+ OzoneConfiguration configuration) {
+ int datanodeCacheMaxSize = configuration.getInt(
+ OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE,
+ OZONE_OM_CONTAINER_LOCATION_DATANODE_CACHE_SIZE_DEFAULT);
+
+ return CacheBuilder.newBuilder()
+ .maximumSize(datanodeCacheMaxSize)
+ .recordStats()
+ .build();
+ }
+
+ static Pipeline newPipelineWithDNCache(Pipeline pipeline,
+ Cache<DatanodeID, DatanodeDetails> datanodeDetailsCache) {
+ Pipeline.Builder builder = pipeline.toBuilder();
+ List<DatanodeDetails> nodes = new ArrayList<>();
+ for (DatanodeDetails node : pipeline.getNodes()) {
+ DatanodeDetails datanodeDetails =
+ datanodeDetailsCache.getIfPresent(node.getID());
+ // Call compareNodeValues to handle IP / hostname changes
+ if (datanodeDetails != null && node.compareNodeValues(datanodeDetails)) {
+ nodes.add(datanodeDetails);
+ } else {
+ datanodeDetailsCache.put(node.getID(), node);
+ nodes.add(node);
+ }
+ }
+ builder.setNodes(nodes);
+ return builder.build();
+ }
+
public ScmBlockLocationProtocol getBlockClient() {
return this.blockClient;
}
@@ -166,6 +212,7 @@ private <T> T handleCacheExecutionException(ExecutionException e)
public void close() {
containerCacheMetrics.unregister();
+ datanodeDetailsCacheMetrics.unregister();
}
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java
index d055ac246c1..89d8c2162c3 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestScmClient.java
@@ -21,6 +21,7 @@
import static java.util.Arrays.asList;
import static org.apache.hadoop.hdds.client.ReplicationConfig.fromTypeAndFactor;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
@@ -28,6 +29,8 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -41,6 +44,7 @@
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -106,9 +110,12 @@ public void testGetContainerLocations(String testCaseName,
throws IOException {
Map<Long, ContainerWithPipeline> actualLocations = new HashMap<>();
-
+ List<DatanodeDetails> dnList = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ dnList.add(randomDatanode());
+ }
for (long containerId : prepopulatedIds) {
- ContainerWithPipeline pipeline = createPipeline(containerId);
+ ContainerWithPipeline pipeline = createPipeline(containerId, dnList);
actualLocations.put(containerId, pipeline);
}
@@ -128,7 +135,7 @@ public void testGetContainerLocations(String testCaseName,
if (!expectedScmCallIds.isEmpty()) {
List<ContainerWithPipeline> scmLocations = new ArrayList<>();
for (long containerId : expectedScmCallIds) {
- ContainerWithPipeline pipeline = createPipeline(containerId);
+ ContainerWithPipeline pipeline = createPipeline(containerId, dnList);
scmLocations.add(pipeline);
actualLocations.put(containerId, pipeline);
}
@@ -166,13 +173,51 @@ public void testGetContainerLocationsWithScmFailures() throws IOException {
assertEquals(runtimeException, actualRt.getCause());
}
- ContainerWithPipeline createPipeline(long containerId) {
+ @Test
+ public void testDatanodeDetailsCacheRecordsStats() {
+ Cache<DatanodeID, DatanodeDetails> datanodeDetailsCache =
+ ScmClient.createDatanodeDetailsCache(new OzoneConfiguration());
+
+ datanodeDetailsCache.getIfPresent(DatanodeID.randomID());
+
+ assertEquals(1, datanodeDetailsCache.stats().missCount());
+ }
+
+ @Test
+ public void testDatanodeDetailsCacheUpdatesIpAddressChange() {
+ Cache<DatanodeID, DatanodeDetails> datanodeDetailsCache =
+ CacheBuilder.newBuilder().build();
+ DatanodeDetails original = randomDatanode();
+ String originalIp = original.getIpAddress();
+ DatanodeDetails updated = DatanodeDetails.newBuilder()
+ .setDatanodeDetails(original)
+ .setIpAddress("updated-ip")
+ .build();
+
+ Pipeline originalPipeline = ScmClient.newPipelineWithDNCache(
+ createPipeline(1L, asList(original)).getPipeline(),
+ datanodeDetailsCache);
+ Pipeline refreshed = ScmClient.newPipelineWithDNCache(
+ createPipeline(2L, asList(updated)).getPipeline(),
+ datanodeDetailsCache);
+
+ assertSame(original, originalPipeline.getNodes().get(0));
+ assertEquals(originalIp, original.getIpAddress());
+ assertEquals(originalIp, originalPipeline.getNodes().get(0).getIpAddress());
+ DatanodeDetails refreshedNode = refreshed.getNodes().get(0);
+ assertSame(updated, refreshedNode);
+ assertEquals("updated-ip", refreshedNode.getIpAddress());
+ assertSame(updated, datanodeDetailsCache.getIfPresent(original.getID()));
+ }
+
+ ContainerWithPipeline createPipeline(long containerId,
+ List<DatanodeDetails> dnList) {
ContainerInfo containerInfo = new ContainerInfo.Builder()
.setContainerID(containerId)
.build();
Pipeline pipeline = Pipeline.newBuilder()
.setId(PipelineID.randomId())
- .setNodes(asList(randomDatanode(), randomDatanode()))
+ .setNodes(dnList)
.setReplicationConfig(fromTypeAndFactor(
ReplicationType.RATIS, ReplicationFactor.THREE))
.setState(Pipeline.PipelineState.OPEN)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org