You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ta...@apache.org on 2024/03/26 00:59:06 UTC

(hadoop) branch branch-3.4 updated (6713a96e5361 -> 0683ba34feef)

This is an automated email from the ASF dual-hosted git repository.

tasanuma pushed a change to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git


    from 6713a96e5361 HDFS-17441. Fix junit dependency by adding missing library in hadoop-hdfs-rbf. (#6669)
     new 317d2db219fe HDFS-17354:Delay invoke clearStaleNamespacesInRouterStateIdContext during router start up (#6498)
     new 0683ba34feef HDFS-17435. Fix TestRouterRpc failed (#6650)

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../server/federation/router/RouterRpcServer.java  |  3 ++
 .../hdfs/server/federation/MockResolver.java       |  9 +++-
 .../server/federation/router/TestDFSRouter.java    | 49 ++++++++++++++++++++++
 3 files changed, 60 insertions(+), 1 deletion(-)


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


(hadoop) 01/02: HDFS-17354:Delay invoke clearStaleNamespacesInRouterStateIdContext during router start up (#6498)

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tasanuma pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 317d2db219fe1c560f778b669772164bf51f2694
Author: Lei313 <47...@users.noreply.github.com>
AuthorDate: Wed Feb 14 08:24:23 2024 +0800

    HDFS-17354:Delay invoke clearStaleNamespacesInRouterStateIdContext during router start up (#6498)
    
    (cherry picked from commit 12498b35bbb754225b0b2ca90d5ad4f5cf628d56)
---
 .../server/federation/router/RouterRpcServer.java  |  3 ++
 .../hdfs/server/federation/MockResolver.java       |  9 ++++-
 .../server/federation/router/TestRouterRpc.java    | 46 ++++++++++++++++++++++
 3 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index 9d7c1263f09b..29aa16ff041e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -430,6 +430,9 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol,
    * Clear expired namespace in the shared RouterStateIdContext.
    */
   private void clearStaleNamespacesInRouterStateIdContext() {
+    if (!router.isRouterState(RouterServiceState.RUNNING)) {
+      return;
+    }
     try {
       final Set<String> resolvedNamespaces = namenodeResolver.getNamespaces()
           .stream()
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
index 554879856ac1..04b9427024c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockResolver.java
@@ -336,7 +336,14 @@ public class MockResolver
   @Override
   public synchronized Set<FederationNamespaceInfo> getNamespaces()
       throws IOException {
-    return Collections.unmodifiableSet(this.namespaces);
+    Set<FederationNamespaceInfo> ret = new TreeSet<>();
+    Set<String> disabled = getDisabledNamespaces();
+    for (FederationNamespaceInfo ns : namespaces) {
+      if (!disabled.contains(ns.getNameserviceId())) {
+        ret.add(ns);
+      }
+    }
+    return Collections.unmodifiableSet(ret);
   }
 
   public void clearDisableNamespaces() {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index 93e905b4eaff..676a8c5fcf9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -23,10 +23,12 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSI
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
 import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS;
 import static org.apache.hadoop.ipc.CallerContext.PROXY_USER_PORT;
 import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -72,6 +74,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.SafeModeAction;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.ha.HAServiceProtocol;
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -115,6 +118,7 @@ import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
 import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
 import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
 import org.apache.hadoop.hdfs.server.federation.metrics.RBFMetrics;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
 import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -2321,4 +2325,46 @@ public class TestRouterRpc {
       fileSystem1.delete(new Path(testPath2), true);
     }
   }
+
+  @Test
+  public void testClearStaleNamespacesInRouterStateIdContext() throws Exception {
+    Router testRouter = new Router();
+    Configuration routerConfig = DFSRouter.getConfiguration();
+    routerConfig.set(FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS, "2000");
+    routerConfig.set(RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE, "false");
+    // Mock resolver classes
+    routerConfig.setClass(RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+        MockResolver.class, ActiveNamenodeResolver.class);
+    routerConfig.setClass(RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+        MockResolver.class, FileSubclusterResolver.class);
+
+    testRouter.init(routerConfig);
+    String nsID1 = cluster.getNameservices().get(0);
+    String nsID2 = cluster.getNameservices().get(1);
+    MockResolver resolver = (MockResolver)testRouter.getNamenodeResolver();
+    resolver.registerNamenode(createNamenodeReport(nsID1, "nn1",
+        HAServiceProtocol.HAServiceState.ACTIVE));
+    resolver.registerNamenode(createNamenodeReport(nsID2, "nn1",
+        HAServiceProtocol.HAServiceState.ACTIVE));
+
+    RouterRpcServer rpcServer = testRouter.getRpcServer();
+
+    rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID1);
+    rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID2);
+
+    resolver.disableNamespace(nsID1);
+    Thread.sleep(3000);
+    RouterStateIdContext context = rpcServer.getRouterStateIdContext();
+    assertEquals(2, context.getNamespaceIdMap().size());
+
+    testRouter.start();
+    Thread.sleep(3000);
+    // wait clear stale namespaces
+    RouterStateIdContext routerStateIdContext = rpcServer.getRouterStateIdContext();
+    int size = routerStateIdContext.getNamespaceIdMap().size();
+    assertEquals(1, size);
+    rpcServer.stop();
+    rpcServer.close();
+    testRouter.close();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


(hadoop) 02/02: HDFS-17435. Fix TestRouterRpc failed (#6650)

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tasanuma pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit 0683ba34feefad6155bf01db8b1e0e448ca988d9
Author: Takanobu Asanuma <ta...@apache.org>
AuthorDate: Tue Mar 26 09:38:35 2024 +0900

    HDFS-17435. Fix TestRouterRpc failed (#6650)
    
    Reviewed-by: Ayush Saxena <ay...@apache.org>
    (cherry picked from commit c4f7a3625b5e51a2800ffd5a24a016d34e267c8a)
---
 .../server/federation/router/TestDFSRouter.java    | 49 ++++++++++++++++++++++
 .../server/federation/router/TestRouterRpc.java    | 46 --------------------
 2 files changed, 49 insertions(+), 46 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDFSRouter.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDFSRouter.java
index 1ab0e9a2ae34..ff279c84d2cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDFSRouter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestDFSRouter.java
@@ -21,8 +21,16 @@ import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
 import org.apache.hadoop.tools.fedbalance.FedBalanceConfigs;
 
+import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport;
+import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS;
+import static org.junit.Assert.assertEquals;
+
 public class TestDFSRouter {
 
   @Test
@@ -36,4 +44,45 @@ public class TestDFSRouter {
     Assert.assertEquals(10, workerThreads);
   }
 
+  @Test
+  public void testClearStaleNamespacesInRouterStateIdContext() throws Exception {
+    Router testRouter = new Router();
+    Configuration routerConfig = DFSRouter.getConfiguration();
+    routerConfig.set(FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS, "2000");
+    routerConfig.set(RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE, "false");
+    // Mock resolver classes
+    routerConfig.setClass(RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
+        MockResolver.class, ActiveNamenodeResolver.class);
+    routerConfig.setClass(RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
+        MockResolver.class, FileSubclusterResolver.class);
+
+    testRouter.init(routerConfig);
+    String nsID1 = "ns0";
+    String nsID2 = "ns1";
+    MockResolver resolver = (MockResolver)testRouter.getNamenodeResolver();
+    resolver.registerNamenode(createNamenodeReport(nsID1, "nn1",
+        HAServiceProtocol.HAServiceState.ACTIVE));
+    resolver.registerNamenode(createNamenodeReport(nsID2, "nn1",
+        HAServiceProtocol.HAServiceState.ACTIVE));
+
+    RouterRpcServer rpcServer = testRouter.getRpcServer();
+
+    rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID1);
+    rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID2);
+
+    resolver.disableNamespace(nsID1);
+    Thread.sleep(3000);
+    RouterStateIdContext context = rpcServer.getRouterStateIdContext();
+    assertEquals(2, context.getNamespaceIdMap().size());
+
+    testRouter.start();
+    Thread.sleep(3000);
+    // wait clear stale namespaces
+    RouterStateIdContext routerStateIdContext = rpcServer.getRouterStateIdContext();
+    int size = routerStateIdContext.getNamespaceIdMap().size();
+    assertEquals(1, size);
+    rpcServer.stop();
+    rpcServer.close();
+    testRouter.close();
+  }
 }
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index 676a8c5fcf9f..93e905b4eaff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -23,12 +23,10 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSI
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.addDirectory;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.countContents;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createFile;
-import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.createNamenodeReport;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.deleteFile;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.getFileStatus;
 import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.verifyFileExists;
 import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST_STRING;
-import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS;
 import static org.apache.hadoop.ipc.CallerContext.PROXY_USER_PORT;
 import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -74,7 +72,6 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.SafeModeAction;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.ha.HAServiceProtocol;
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -118,7 +115,6 @@ import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
 import org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCMetrics;
 import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
 import org.apache.hadoop.hdfs.server.federation.metrics.RBFMetrics;
-import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
 import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -2325,46 +2321,4 @@ public class TestRouterRpc {
       fileSystem1.delete(new Path(testPath2), true);
     }
   }
-
-  @Test
-  public void testClearStaleNamespacesInRouterStateIdContext() throws Exception {
-    Router testRouter = new Router();
-    Configuration routerConfig = DFSRouter.getConfiguration();
-    routerConfig.set(FEDERATION_STORE_MEMBERSHIP_EXPIRATION_MS, "2000");
-    routerConfig.set(RBFConfigKeys.DFS_ROUTER_SAFEMODE_ENABLE, "false");
-    // Mock resolver classes
-    routerConfig.setClass(RBFConfigKeys.FEDERATION_NAMENODE_RESOLVER_CLIENT_CLASS,
-        MockResolver.class, ActiveNamenodeResolver.class);
-    routerConfig.setClass(RBFConfigKeys.FEDERATION_FILE_RESOLVER_CLIENT_CLASS,
-        MockResolver.class, FileSubclusterResolver.class);
-
-    testRouter.init(routerConfig);
-    String nsID1 = cluster.getNameservices().get(0);
-    String nsID2 = cluster.getNameservices().get(1);
-    MockResolver resolver = (MockResolver)testRouter.getNamenodeResolver();
-    resolver.registerNamenode(createNamenodeReport(nsID1, "nn1",
-        HAServiceProtocol.HAServiceState.ACTIVE));
-    resolver.registerNamenode(createNamenodeReport(nsID2, "nn1",
-        HAServiceProtocol.HAServiceState.ACTIVE));
-
-    RouterRpcServer rpcServer = testRouter.getRpcServer();
-
-    rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID1);
-    rpcServer.getRouterStateIdContext().getNamespaceStateId(nsID2);
-
-    resolver.disableNamespace(nsID1);
-    Thread.sleep(3000);
-    RouterStateIdContext context = rpcServer.getRouterStateIdContext();
-    assertEquals(2, context.getNamespaceIdMap().size());
-
-    testRouter.start();
-    Thread.sleep(3000);
-    // wait clear stale namespaces
-    RouterStateIdContext routerStateIdContext = rpcServer.getRouterStateIdContext();
-    int size = routerStateIdContext.getNamespaceIdMap().size();
-    assertEquals(1, size);
-    rpcServer.stop();
-    rpcServer.close();
-    testRouter.close();
-  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org