You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2019/11/15 15:40:39 UTC
[hadoop] branch branch-2 updated: HDFS-14979 Allow Balancer to
submit getBlocks calls to Observer Nodes when possible. Contributed by Erik
Krogen.
This is an automated email from the ASF dual-hosted git repository.
xkrogen pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 792e2ae HDFS-14979 Allow Balancer to submit getBlocks calls to Observer Nodes when possible. Contributed by Erik Krogen.
792e2ae is described below
commit 792e2aecb3d81ce478f2a44350e4a64351967e52
Author: Erik Krogen <xk...@apache.org>
AuthorDate: Mon Nov 11 14:32:51 2019 -0800
HDFS-14979 Allow Balancer to submit getBlocks calls to Observer Nodes when possible. Contributed by Erik Krogen.
(cherry picked from 586defe7113ed246ed0275bb3833882a3d873d70)
(cherry picked from dec765b329d3947f30273c0e7f0c4eb607ec42c9)
(cherry picked from 5cf36aa2b19aad8677e1a7553ff1f05805f772b7)
---
.../hdfs/server/protocol/NamenodeProtocol.java | 2 ++
.../balancer/TestBalancerWithHANameNodes.java | 22 ++++++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
index ccdf516c..f218e8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
+import org.apache.hadoop.hdfs.server.namenode.ha.ReadOnly;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.security.KerberosInfo;
@@ -76,6 +77,7 @@ public interface NamenodeProtocol {
datanode does not exist
*/
@Idempotent
+ @ReadOnly
public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
throws IOException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
index 4a398db..779e19c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
@@ -18,8 +18,14 @@
package org.apache.hadoop.hdfs.server.balancer;
import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
import java.net.URI;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -32,9 +38,13 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology.NNConf;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider;
+import org.mockito.Matchers;
import org.junit.Test;
/**
@@ -128,12 +138,24 @@ public class TestBalancerWithHANameNodes {
cluster = qjmhaCluster.getDfsCluster();
cluster.waitClusterUp();
cluster.waitActive();
+ List<FSNamesystem> namesystemSpies = new ArrayList<>();
+ for (int i = 0; i < cluster.getNumNameNodes(); i++) {
+ namesystemSpies.add(
+ NameNodeAdapter.spyOnNamesystem(cluster.getNameNode(i)));
+ }
DistributedFileSystem dfs = HATestUtil.configureObserverReadFs(
cluster, conf, ObserverReadProxyProvider.class, true);
client = dfs.getClient().getNamenode();
doTest(conf);
+ for (int i = 0; i < cluster.getNumNameNodes(); i++) {
+ // First observer node is at idx 2 so it should get both getBlocks calls
+ // all other NameNodes should see 0 getBlocks calls
+ int expectedCount = (i == 2) ? 2 : 0;
+ verify(namesystemSpies.get(i), times(expectedCount))
+ .getBlocks(Matchers.<DatanodeID>any(), anyLong());
+ }
} finally {
if (qjmhaCluster != null) {
qjmhaCluster.shutdown();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org