You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sz...@apache.org on 2022/11/28 23:46:39 UTC
[ozone] branch master updated: HDDS-7500. Ozone recon server Memory Optimization. (#3974)
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8db02c864b HDDS-7500. Ozone recon server Memory Optimization. (#3974)
8db02c864b is described below
commit 8db02c864b568805a6b12929e8808ef5b9b6a292
Author: devmadhuu <de...@hotmail.com>
AuthorDate: Tue Nov 29 05:16:34 2022 +0530
HDDS-7500. Ozone recon server Memory Optimization. (#3974)
---
.../common/src/main/resources/ozone-default.xml | 11 +++++++
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 6 ++++
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 3 +-
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 35 ++++++++++++++++++++--
.../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 33 +++++++++++++-------
.../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 21 +++++++------
.../recon/api/TestNSSummaryEndpointWithFSO.java | 7 ++++-
.../recon/tasks/TestNSSummaryTaskWithFSO.java | 10 ++++++-
8 files changed, 101 insertions(+), 25 deletions(-)
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7d035d223f..fe11a3b72e 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -3422,4 +3422,15 @@
optimum performance.
</description>
</property>
+
+ <property>
+ <name>ozone.recon.nssummary.flush.db.max.threshold</name>
+ <value>150000</value>
+ <tag>OZONE, RECON, PERFORMANCE</tag>
+ <description>
+ Maximum threshold number of entries to hold in memory for NSSummary task in hashmap before flushing to
+ recon rocks DB namespaceSummaryTable
+ </description>
+ </property>
+
</configuration>
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
index fb00898cc0..c9e31563d3 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServerConfigKeys.java
@@ -143,6 +143,12 @@ public final class ReconServerConfigKeys {
"ozone.recon.scm.connection.request.timeout";
public static final String
OZONE_RECON_SCM_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "5s";
+
+ public static final String OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD =
+ "ozone.recon.nssummary.flush.db.max.threshold";
+
+ public static final long
+ OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD_DEFAULT = 150 * 1000L;
/**
* Private constructor for utility class.
*/
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java
index 63b6ee375c..6adf71586f 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java
@@ -79,7 +79,8 @@ public class NSSummaryTask implements ReconOmTask {
this.reconOMMetadataManager = reconOMMetadataManager;
this.ozoneConfiguration = ozoneConfiguration;
this.nsSummaryTaskWithFSO = new NSSummaryTaskWithFSO(
- reconNamespaceSummaryManager, reconOMMetadataManager);
+ reconNamespaceSummaryManager,
+ reconOMMetadataManager, ozoneConfiguration);
this.nsSummaryTaskWithLegacy = new NSSummaryTaskWithLegacy(
reconNamespaceSummaryManager,
reconOMMetadataManager, ozoneConfiguration);
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java
index 4cadbf273a..648f681100 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskDbEventHandler.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.recon.tasks;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.db.RDBBatchOperation;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -31,6 +32,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD_DEFAULT;
+
/**
* Class for holding all NSSummaryTask methods
* related to DB operations so that they can commonly be
@@ -40,16 +44,22 @@ public class NSSummaryTaskDbEventHandler {
private static final Logger LOG =
LoggerFactory.getLogger(NSSummaryTaskDbEventHandler.class);
-
private ReconNamespaceSummaryManager reconNamespaceSummaryManager;
private ReconOMMetadataManager reconOMMetadataManager;
+ private final long nsSummaryFlushToDBMaxThreshold;
+
public NSSummaryTaskDbEventHandler(ReconNamespaceSummaryManager
reconNamespaceSummaryManager,
ReconOMMetadataManager
- reconOMMetadataManager) {
+ reconOMMetadataManager,
+ OzoneConfiguration
+ ozoneConfiguration) {
this.reconNamespaceSummaryManager = reconNamespaceSummaryManager;
this.reconOMMetadataManager = reconOMMetadataManager;
+ nsSummaryFlushToDBMaxThreshold = ozoneConfiguration.getLong(
+ OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD,
+ OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD_DEFAULT);
}
public ReconNamespaceSummaryManager getReconNamespaceSummaryManager() {
@@ -194,4 +204,25 @@ public class NSSummaryTaskDbEventHandler {
nsSummary.removeChildDir(objectId);
nsSummaryMap.put(parentObjectId, nsSummary);
}
+
+ protected boolean flushAndCommitNSToDB(Map<Long, NSSummary> nsSummaryMap) {
+ try {
+ writeNSSummariesToDB(nsSummaryMap);
+ nsSummaryMap.clear();
+ } catch (IOException e) {
+ LOG.error("Unable to write Namespace Summary data in Recon DB.", e);
+ return false;
+ }
+ return true;
+ }
+
+ protected boolean checkAndCallFlushToDB(
+ Map<Long, NSSummary> nsSummaryMap) {
+ // if map contains more than entries, flush to DB and clear the map
+ if (null != nsSummaryMap && nsSummaryMap.size() >=
+ nsSummaryFlushToDBMaxThreshold) {
+ return flushAndCommitNSToDB(nsSummaryMap);
+ }
+ return true;
+ }
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java
index 0f80927d83..9806b198b2 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.recon.tasks;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -50,8 +51,11 @@ public class NSSummaryTaskWithFSO extends NSSummaryTaskDbEventHandler {
public NSSummaryTaskWithFSO(ReconNamespaceSummaryManager
reconNamespaceSummaryManager,
ReconOMMetadataManager
- reconOMMetadataManager) {
- super(reconNamespaceSummaryManager, reconOMMetadataManager);
+ reconOMMetadataManager,
+ OzoneConfiguration
+ ozoneConfiguration) {
+ super(reconNamespaceSummaryManager,
+ reconOMMetadataManager, ozoneConfiguration);
}
// We only listen to updates from FSO-enabled KeyTable(FileTable) and DirTable
@@ -110,6 +114,7 @@ public class NSSummaryTaskWithFSO extends NSSummaryTaskDbEventHandler {
LOG.debug("Skipping DB update event : {}",
omdbUpdateEvent.getAction());
}
+
} else {
// directory update on DirTable
OMDBUpdateEvent<String, OmDirectoryInfo> dirTableUpdateEvent =
@@ -147,12 +152,13 @@ public class NSSummaryTaskWithFSO extends NSSummaryTaskDbEventHandler {
ioEx);
return false;
}
+ if (!checkAndCallFlushToDB(nsSummaryMap)) {
+ return false;
+ }
}
- try {
- writeNSSummariesToDB(nsSummaryMap);
- } catch (IOException e) {
- LOG.error("Unable to write Namespace Summary data in Recon DB.", e);
+ // flush and commit left out entries at end
+ if (!flushAndCommitNSToDB(nsSummaryMap)) {
return false;
}
@@ -173,6 +179,9 @@ public class NSSummaryTaskWithFSO extends NSSummaryTaskDbEventHandler {
Table.KeyValue<String, OmDirectoryInfo> kv = dirTableIter.next();
OmDirectoryInfo directoryInfo = kv.getValue();
handlePutDirEvent(directoryInfo, nsSummaryMap);
+ if (!checkAndCallFlushToDB(nsSummaryMap)) {
+ return false;
+ }
}
}
@@ -186,6 +195,9 @@ public class NSSummaryTaskWithFSO extends NSSummaryTaskDbEventHandler {
Table.KeyValue<String, OmKeyInfo> kv = keyTableIter.next();
OmKeyInfo keyInfo = kv.getValue();
handlePutKeyEvent(keyInfo, nsSummaryMap);
+ if (!checkAndCallFlushToDB(nsSummaryMap)) {
+ return false;
+ }
}
}
@@ -194,14 +206,13 @@ public class NSSummaryTaskWithFSO extends NSSummaryTaskDbEventHandler {
ioEx);
return false;
}
-
- try {
- writeNSSummariesToDB(nsSummaryMap);
- } catch (IOException e) {
- LOG.error("Unable to write Namespace Summary data in Recon DB.", e);
+ // flush and commit left out keys at end
+ if (!flushAndCommitNSToDB(nsSummaryMap)) {
return false;
}
LOG.info("Completed a reprocess run of NSSummaryTaskWithFSO");
return true;
}
+
+
}
diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java
index 6e414a3b4e..a2a9598261 100644
--- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java
+++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithLegacy.java
@@ -60,7 +60,8 @@ public class NSSummaryTaskWithLegacy extends NSSummaryTaskDbEventHandler {
reconOMMetadataManager,
OzoneConfiguration
ozoneConfiguration) {
- super(reconNamespaceSummaryManager, reconOMMetadataManager);
+ super(reconNamespaceSummaryManager,
+ reconOMMetadataManager, ozoneConfiguration);
// true if FileSystemPaths enabled
enableFileSystemPaths = ozoneConfiguration
.getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
@@ -185,12 +186,13 @@ public class NSSummaryTaskWithLegacy extends NSSummaryTaskDbEventHandler {
ioEx);
return false;
}
+ if (!checkAndCallFlushToDB(nsSummaryMap)) {
+ return false;
+ }
}
- try {
- writeNSSummariesToDB(nsSummaryMap);
- } catch (IOException e) {
- LOG.error("Unable to write Namespace Summary data in Recon DB.", e);
+ // flush and commit left out entries at end
+ if (!flushAndCommitNSToDB(nsSummaryMap)) {
return false;
}
@@ -241,6 +243,9 @@ public class NSSummaryTaskWithLegacy extends NSSummaryTaskDbEventHandler {
} else {
handlePutKeyEvent(keyInfo, nsSummaryMap);
}
+ if (!checkAndCallFlushToDB(nsSummaryMap)) {
+ return false;
+ }
}
}
} catch (IOException ioEx) {
@@ -249,10 +254,8 @@ public class NSSummaryTaskWithLegacy extends NSSummaryTaskDbEventHandler {
return false;
}
- try {
- writeNSSummariesToDB(nsSummaryMap);
- } catch (IOException e) {
- LOG.error("Unable to write Namespace Summary data in Recon DB.", e);
+ // flush and commit left out entries at end
+ if (!flushAndCommitNSToDB(nsSummaryMap)) {
return false;
}
LOG.info("Completed a reprocess run of NSSummaryTaskWithLegacy");
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java
index c68bab8735..cdfb3ab28f 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestNSSummaryEndpointWithFSO.java
@@ -75,6 +75,7 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirT
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm;
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager;
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -107,6 +108,7 @@ public class TestNSSummaryEndpointWithFSO {
private ReconOMMetadataManager reconOMMetadataManager;
private NSSummaryEndpoint nsSummaryEndpoint;
+ private OzoneConfiguration ozoneConfiguration;
private static final String TEST_PATH_UTILITY =
"/vol1/buck1/a/b/c/d/e/file1.txt";
@@ -341,6 +343,9 @@ public class TestNSSummaryEndpointWithFSO {
@Before
public void setUp() throws Exception {
+ ozoneConfiguration = new OzoneConfiguration();
+ ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD,
+ 10);
OMMetadataManager omMetadataManager = initializeNewOmMetadataManager(
temporaryFolder.newFolder());
OzoneManagerServiceProviderImpl ozoneManagerServiceProvider =
@@ -368,7 +373,7 @@ public class TestNSSummaryEndpointWithFSO {
populateOMDB();
NSSummaryTaskWithFSO nSSummaryTaskWithFso =
new NSSummaryTaskWithFSO(reconNamespaceSummaryManager,
- reconOMMetadataManager);
+ reconOMMetadataManager, ozoneConfiguration);
nSSummaryTaskWithFso.reprocessWithFSO(reconOMMetadataManager);
}
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java
index 6b6b831c06..b98f03299b 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestNSSummaryTaskWithFSO.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.recon.tasks;
import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.utils.db.RDBBatchOperation;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -50,6 +51,7 @@ import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestRe
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm;
import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm;
+import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD;
/**
* Test for NSSummaryTaskWithFSO.
@@ -65,6 +67,8 @@ public final class TestNSSummaryTaskWithFSO {
private static ReconOMMetadataManager reconOMMetadataManager;
private static NSSummaryTaskWithFSO nSSummaryTaskWithFso;
+ private static OzoneConfiguration ozoneConfiguration;
+
// Object names in FSO-enabled format
private static final String VOL = "vol";
private static final String BUCKET_ONE = "bucket1";
@@ -117,6 +121,9 @@ public final class TestNSSummaryTaskWithFSO {
@BeforeClass
public static void setUp() throws Exception {
+ ozoneConfiguration = new OzoneConfiguration();
+ ozoneConfiguration.setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD,
+ 10);
omMetadataManager = initializeNewOmMetadataManager(
TEMPORARY_FOLDER.newFolder());
OzoneManagerServiceProvider ozoneManagerServiceProvider =
@@ -141,7 +148,8 @@ public final class TestNSSummaryTaskWithFSO {
populateOMDB();
nSSummaryTaskWithFso = new NSSummaryTaskWithFSO(
- reconNamespaceSummaryManager, reconOMMetadataManager);
+ reconNamespaceSummaryManager, reconOMMetadataManager,
+ ozoneConfiguration);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org