You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ra...@apache.org on 2018/01/24 07:32:09 UTC
[14/50] [abbrv] hadoop git commit: HDFS-11338: [SPS]: Fix timeout
issue in unit tests caused by longger NN down time. Contributed by Wei Zhou
and Rakesh R
HDFS-11338: [SPS]: Fix timeout issue in unit tests caused by longger NN down time. Contributed by Wei Zhou and Rakesh R
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db3cc98f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db3cc98f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db3cc98f
Branch: refs/heads/HDFS-10285
Commit: db3cc98fdf8be7241d120af71131cbcc62935e07
Parents: 46d56ba
Author: Uma Maheswara Rao G <um...@intel.com>
Authored: Tue Apr 11 14:25:01 2017 -0700
Committer: Rakesh Radhakrishnan <ra...@apache.org>
Committed: Wed Jan 24 11:13:05 2018 +0530
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 13 +++++--
.../BlockStorageMovementAttemptedItems.java | 25 +++++++++----
.../hdfs/server/namenode/FSNamesystem.java | 2 +-
.../server/namenode/StoragePolicySatisfier.java | 38 ++++++++++++++------
.../TestBlockStorageMovementAttemptedItems.java | 3 +-
5 files changed, 60 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db3cc98f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 0c7b982..dd491cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -735,7 +735,7 @@ public class BlockManager implements BlockStatsMXBean {
public void close() {
if (sps != null) {
- sps.stop(false);
+ sps.deactivate(false);
}
bmSafeMode.close();
try {
@@ -750,6 +750,7 @@ public class BlockManager implements BlockStatsMXBean {
datanodeManager.close();
pendingReconstruction.stop();
blocksMap.close();
+ stopSPSGracefully();
}
/** @return the datanodeManager */
@@ -5060,10 +5061,18 @@ public class BlockManager implements BlockStatsMXBean {
LOG.info("Storage policy satisfier is already stopped.");
return;
}
- sps.stop(true);
+ sps.deactivate(true);
}
/**
+ * Timed wait to stop storage policy satisfier daemon threads.
+ */
+ public void stopSPSGracefully() {
+ if (sps != null) {
+ sps.stopGracefully();
+ }
+ }
+ /**
* @return True if storage policy satisfier running.
*/
public boolean isStoragePolicySatisfierRunning() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db3cc98f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
index f15db73..26b98d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BlockStorageMovementAttemptedItems.java
@@ -130,21 +130,34 @@ public class BlockStorageMovementAttemptedItems {
}
/**
- * Stops the monitor thread.
+ * Sets running flag to false. Also, this will interrupt monitor thread and
+ * clear all the queued up tasks.
*/
- public synchronized void stop() {
+ public synchronized void deactivate() {
monitorRunning = false;
if (timerThread != null) {
timerThread.interrupt();
- try {
- timerThread.join(3000);
- } catch (InterruptedException ie) {
- }
}
this.clearQueues();
}
/**
+ * Timed wait to stop monitor thread.
+ */
+ synchronized void stopGracefully() {
+ if (timerThread == null) {
+ return;
+ }
+ if (monitorRunning) {
+ deactivate();
+ }
+ try {
+ timerThread.join(3000);
+ } catch (InterruptedException ie) {
+ }
+ }
+
+ /**
* This class contains information of an attempted trackID. Information such
* as, (a)last attempted time stamp, (b)whether all the blocks in the trackID
* were attempted and blocks movement has been scheduled to satisfy storage
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db3cc98f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 53aefdb..ea06018 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1320,7 +1320,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (blockManager != null) {
blockManager.deactivateSPS();
}
-
stopSecretManager();
leaseManager.stopMonitor();
if (nnrmthread != null) {
@@ -1359,6 +1358,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// Don't want to keep replication queues when not in Active.
blockManager.clearQueues();
blockManager.setInitializedReplQueues(false);
+ blockManager.stopSPSGracefully();
}
} finally {
writeUnlock("stopActiveServices");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db3cc98f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
index 337d5b5..8cf9920 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySatisfier.java
@@ -115,22 +115,21 @@ public class StoragePolicySatisfier implements Runnable {
}
/**
- * Stop storage policy satisfier demon thread.
+ * Deactivates storage policy satisfier by stopping its services.
*
- * @param reconfigStop
+ * @param reconfig
+ * true represents deactivating SPS service as requested by admin,
+ * false otherwise
*/
- public synchronized void stop(boolean reconfigStop) {
+ public synchronized void deactivate(boolean reconfig) {
isRunning = false;
if (storagePolicySatisfierThread == null) {
return;
}
+
storagePolicySatisfierThread.interrupt();
- try {
- storagePolicySatisfierThread.join(3000);
- } catch (InterruptedException ie) {
- }
- this.storageMovementsMonitor.stop();
- if (reconfigStop) {
+ this.storageMovementsMonitor.deactivate();
+ if (reconfig) {
LOG.info("Stopping StoragePolicySatisfier, as admin requested to "
+ "deactivate it.");
this.clearQueuesWithNotification();
@@ -141,6 +140,23 @@ public class StoragePolicySatisfier implements Runnable {
}
/**
+ * Timed wait to stop storage policy satisfier daemon threads.
+ */
+ public synchronized void stopGracefully() {
+ if (isRunning) {
+ deactivate(true);
+ }
+ this.storageMovementsMonitor.stopGracefully();
+ if (storagePolicySatisfierThread == null) {
+ return;
+ }
+ try {
+ storagePolicySatisfierThread.join(3000);
+ } catch (InterruptedException ie) {
+ }
+ }
+
+ /**
* Check whether StoragePolicySatisfier is running.
* @return true if running
*/
@@ -162,7 +178,7 @@ public class StoragePolicySatisfier implements Runnable {
if (!isRunning) {
// Stopping monitor thread and clearing queues as well
this.clearQueues();
- this.storageMovementsMonitor.stop();
+ this.storageMovementsMonitor.stopGracefully();
LOG.error(
"Stopping StoragePolicySatisfier thread " + "as Mover ID file "
+ HdfsServerConstants.MOVER_ID_PATH.toString()
@@ -194,7 +210,7 @@ public class StoragePolicySatisfier implements Runnable {
isRunning = false;
// Stopping monitor thread and clearing queues as well
this.clearQueues();
- this.storageMovementsMonitor.stop();
+ this.storageMovementsMonitor.stopGracefully();
}
if (!namesystem.isRunning()) {
LOG.info("Stopping StoragePolicySatisfier.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db3cc98f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java
index 95142d3..8c7d982 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockStorageMovementAttemptedItems.java
@@ -47,7 +47,8 @@ public class TestBlockStorageMovementAttemptedItems {
@After
public void teardown() {
if (bsmAttemptedItems != null) {
- bsmAttemptedItems.stop();
+ bsmAttemptedItems.deactivate();
+ bsmAttemptedItems.stopGracefully();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org