You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ra...@apache.org on 2017/08/27 07:15:56 UTC
[29/50] [abbrv] hadoop git commit: HDFS-11186. [SPS]: Daemon thread
of SPS should start only in Active NN. Contributed by Wei Zhou
HDFS-11186. [SPS]: Daemon thread of SPS should start only in Active NN. Contributed by Wei Zhou
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b775bdb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b775bdb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b775bdb
Branch: refs/heads/HDFS-10285
Commit: 8b775bdb7759a0cb035762429812246027ed1ae0
Parents: 498e4de
Author: Rakesh Radhakrishnan <ra...@apache.org>
Authored: Thu Jan 12 09:01:30 2017 +0530
Committer: Rakesh Radhakrishnan <ra...@apache.org>
Committed: Sun Aug 27 11:55:05 2017 +0530
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 9 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 11 +-
.../hdfs/server/namenode/FSNamesystem.java | 6 +
.../hadoop/hdfs/server/namenode/NameNode.java | 7 ++
.../TestStoragePolicySatisfierWithHA.java | 109 +++++++++++++++++++
5 files changed, 138 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b775bdb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index fed1506..f053265 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -427,9 +427,15 @@ public class BlockManager implements BlockStatsMXBean {
*/
private final short minReplicationToBeInMaintenance;
+ /**
+ * Whether HA is enabled.
+ */
+ private final boolean haEnabled;
+
public BlockManager(final Namesystem namesystem, boolean haEnabled,
final Configuration conf) throws IOException {
this.namesystem = namesystem;
+ this.haEnabled = haEnabled;
datanodeManager = new DatanodeManager(this, namesystem, conf);
heartbeatManager = datanodeManager.getHeartbeatManager();
this.blockIdManager = new BlockIdManager(this);
@@ -694,7 +700,7 @@ public class BlockManager implements BlockStatsMXBean {
this.blockReportThread.start();
mxBeanName = MBeans.register("NameNode", "BlockStats", this);
bmSafeMode.activate(blockTotal);
- if (sps != null) {
+ if (sps != null && !haEnabled) {
sps.start();
}
}
@@ -4898,6 +4904,7 @@ public class BlockManager implements BlockStatsMXBean {
LOG.info("Storage policy satisfier is already running.");
return;
}
+
sps.start();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b775bdb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index dd6f90e..d24b336 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NetworkTopology;
@@ -661,9 +662,13 @@ public class Mover {
try {
spsRunning = nnc.getDistributedFileSystem().getClient()
.isStoragePolicySatisfierRunning();
- } catch (StandbyException e) {
- System.err.println("Skip Standby Namenode. " + nnc.toString());
- continue;
+ } catch (RemoteException e) {
+ IOException cause = e.unwrapRemoteException();
+ if (cause instanceof StandbyException) {
+ System.err.println("Skip Standby Namenode. " + nnc.toString());
+ continue;
+ }
+ throw e;
}
if (spsRunning) {
System.err.println("Mover failed due to StoragePolicySatisfier"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b775bdb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index a5f5eaa..3cb47b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1274,6 +1274,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
FSDirEncryptionZoneOp.warmUpEdekCache(edekCacheLoader, dir,
edekCacheLoaderDelay, edekCacheLoaderInterval);
}
+
+ blockManager.activateSPS();
} finally {
startingActiveService = false;
blockManager.checkSafeMode();
@@ -1303,6 +1305,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
LOG.info("Stopping services started for active state");
writeLock();
try {
+ if (blockManager != null) {
+ blockManager.deactivateSPS();
+ }
+
stopSecretManager();
leaseManager.stopMonitor();
if (nnrmthread != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b775bdb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 27c07e2..7cc46f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -2132,6 +2132,13 @@ public class NameNode extends ReconfigurableBase implements
+ "we must pass true/false only"));
}
+ if (!isActiveState()) {
+ throw new ReconfigurationException(property, newVal,
+ getConf().get(property), new HadoopIllegalArgumentException(
+ "Activating or deactivating storage policy satisfier service on "
+ + state + " NameNode is not allowed"));
+ }
+
boolean activateSPS = Boolean.parseBoolean(newVal);
if (activateSPS) {
namesystem.getBlockManager().activateSPS();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b775bdb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithHA.java
new file mode 100644
index 0000000..4d226ff
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithHA.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.ReconfigurationException;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Tests that StoragePolicySatisfier is able to work with HA enabled.
+ */
+public class TestStoragePolicySatisfierWithHA {
+ private MiniDFSCluster cluster = null;
+
+ @Before
+ public void setUp() throws IOException {
+ Configuration conf = new Configuration();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .numDataNodes(1)
+ .build();
+ }
+
+ /**
+ * Tests to verify that SPS should run/stop automatically when NN state
+ * changes between Standby and Active.
+ */
+ @Test(timeout = 100000)
+ public void testWhenNNHAStateChanges() throws IOException {
+ try {
+ DistributedFileSystem fs;
+ boolean running;
+
+ cluster.waitActive();
+ fs = cluster.getFileSystem(0);
+
+ try {
+ fs.getClient().isStoragePolicySatisfierRunning();
+ Assert.fail("Call this function to Standby NN should "
+ + "raise an exception.");
+ } catch (RemoteException e) {
+ IOException cause = e.unwrapRemoteException();
+ if (!(cause instanceof StandbyException)) {
+ Assert.fail("Unexpected exception happened " + e);
+ }
+ }
+
+ cluster.transitionToActive(0);
+ running = fs.getClient().isStoragePolicySatisfierRunning();
+ Assert.assertTrue("StoragePolicySatisfier should be active "
+ + "when NN transits from Standby to Active mode.", running);
+
+ // NN transits from Active to Standby
+ cluster.transitionToStandby(0);
+ try {
+ fs.getClient().isStoragePolicySatisfierRunning();
+ Assert.fail("NN in Standby again, call this function should "
+ + "raise an exception.");
+ } catch (RemoteException e) {
+ IOException cause = e.unwrapRemoteException();
+ if (!(cause instanceof StandbyException)) {
+ Assert.fail("Unexpected exception happened " + e);
+ }
+ }
+
+ try {
+ cluster.getNameNode(0).reconfigurePropertyImpl(
+ DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ACTIVATE_KEY, "false");
+ Assert.fail("It's not allowed to activate or deactivate"
+ + " StoragePolicySatisfier on Standby NameNode");
+ } catch (ReconfigurationException e) {
+ GenericTestUtils.assertExceptionContains("Could not change property "
+ + DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_ACTIVATE_KEY
+ + " from 'true' to 'false'", e);
+ GenericTestUtils.assertExceptionContains(
+ "Activating or deactivating storage policy satisfier service on "
+ + "standby NameNode is not allowed", e.getCause());
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org