You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by um...@apache.org on 2018/08/10 20:26:51 UTC
[39/50] [abbrv] hadoop git commit: HDFS-13097: [SPS]: Fix the branch
review comments(Part1). Contributed by Surendra Singh.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5262e30/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
index 85a101f..47ea39f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeReconfigure.java
@@ -252,8 +252,8 @@ public class TestNameNodeReconfigure {
// Since DFS_STORAGE_POLICY_ENABLED_KEY is disabled, SPS can't be enabled.
assertEquals("SPS shouldn't start as "
+ DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY + " is disabled", false,
- nameNode.getNamesystem().getBlockManager()
- .isStoragePolicySatisfierRunning());
+ nameNode.getNamesystem().getBlockManager().getSPSManager()
+ .isInternalSatisfierRunning());
verifySPSEnabled(nameNode, DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
StoragePolicySatisfierMode.INTERNAL, false);
@@ -280,8 +280,8 @@ public class TestNameNodeReconfigure {
fail("ReconfigurationException expected");
} catch (ReconfigurationException e) {
GenericTestUtils.assertExceptionContains(
- "For enabling or disabling storage policy satisfier, we must "
- + "pass either none/internal/external string value only",
+ "For enabling or disabling storage policy satisfier, must "
+ + "pass either internal/external/none string value only",
e.getCause());
}
@@ -301,8 +301,8 @@ public class TestNameNodeReconfigure {
nameNode.reconfigureProperty(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
StoragePolicySatisfierMode.EXTERNAL.toString());
assertEquals(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY + " has wrong value",
- false, nameNode.getNamesystem().getBlockManager()
- .isStoragePolicySatisfierRunning());
+ false, nameNode.getNamesystem().getBlockManager().getSPSManager()
+ .isInternalSatisfierRunning());
assertEquals(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY + " has wrong value",
StoragePolicySatisfierMode.EXTERNAL.toString(),
nameNode.getConf().get(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
@@ -342,8 +342,8 @@ public class TestNameNodeReconfigure {
nameNode.reconfigureProperty(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
StoragePolicySatisfierMode.INTERNAL.toString());
assertEquals(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY + " has wrong value",
- true, nameNode.getNamesystem().getBlockManager()
- .isStoragePolicySatisfierRunning());
+ true, nameNode.getNamesystem().getBlockManager().getSPSManager()
+ .isInternalSatisfierRunning());
assertEquals(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY + " has wrong value",
StoragePolicySatisfierMode.INTERNAL.toString(),
nameNode.getConf().get(DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
@@ -353,7 +353,8 @@ public class TestNameNodeReconfigure {
void verifySPSEnabled(final NameNode nameNode, String property,
StoragePolicySatisfierMode expected, boolean isSatisfierRunning) {
assertEquals(property + " has wrong value", isSatisfierRunning, nameNode
- .getNamesystem().getBlockManager().isStoragePolicySatisfierRunning());
+ .getNamesystem().getBlockManager().getSPSManager()
+ .isInternalSatisfierRunning());
String actual = nameNode.getConf().get(property,
DFS_STORAGE_POLICY_SATISFIER_MODE_DEFAULT);
assertEquals(property + " has wrong value", expected,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5262e30/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
index b84214c..9f98777 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPersistentStoragePolicySatisfier.java
@@ -389,7 +389,8 @@ public class TestPersistentStoragePolicySatisfier {
fs.setStoragePolicy(testFile, ONE_SSD);
fs.satisfyStoragePolicy(testFile);
- cluster.getNamesystem().getBlockManager().disableSPS();
+ cluster.getNamesystem().getBlockManager().getSPSManager()
+ .changeModeEvent(StoragePolicySatisfierMode.NONE);
// Make sure satisfy xattr has been removed.
DFSTestUtil.waitForXattrRemoved(testFileName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5262e30/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithHA.java
index e89cfa3..b3734d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfierWithHA.java
@@ -103,7 +103,7 @@ public class TestStoragePolicySatisfierWithHA {
dfs = cluster.getFileSystem(1);
try {
- dfs.getClient().isStoragePolicySatisfierRunning();
+ dfs.getClient().isInternalSatisfierRunning();
Assert.fail("Call this function to Standby NN should "
+ "raise an exception.");
} catch (RemoteException e) {
@@ -115,14 +115,14 @@ public class TestStoragePolicySatisfierWithHA {
cluster.transitionToActive(0);
dfs = cluster.getFileSystem(0);
- running = dfs.getClient().isStoragePolicySatisfierRunning();
+ running = dfs.getClient().isInternalSatisfierRunning();
Assert.assertTrue("StoragePolicySatisfier should be active "
+ "when NN transits from Standby to Active mode.", running);
// NN transits from Active to Standby
cluster.transitionToStandby(0);
try {
- dfs.getClient().isStoragePolicySatisfierRunning();
+ dfs.getClient().isInternalSatisfierRunning();
Assert.fail("NN in Standby again, call this function should "
+ "raise an exception.");
} catch (RemoteException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5262e30/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
index 9e0a39f..6f7fe89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfier.java
@@ -445,15 +445,10 @@ public class TestStoragePolicySatisfier {
try {
hdfsAdmin.satisfyStoragePolicy(new Path(FILE));
hdfsAdmin.satisfyStoragePolicy(new Path(FILE));
- Assert.fail(String.format("Should failed to satisfy storage policy "
- + "for %s ,since it has been " + "added to satisfy movement queue.",
- FILE));
- } catch (IOException e) {
- GenericTestUtils.assertExceptionContains(
- String.format("Cannot request to call satisfy storage policy "
- + "on path %s, as this file/dir was already called for "
- + "satisfying storage policy.", FILE),
- e);
+ } catch (Exception e) {
+ Assert.fail(String.format("Allow to invoke mutlipe times "
+ + "#satisfyStoragePolicy() api for a path %s , internally just "
+ + "skipping addtion to satisfy movement queue.", FILE));
}
} finally {
shutdownCluster();
@@ -563,7 +558,7 @@ public class TestStoragePolicySatisfier {
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
StoragePolicySatisfierMode.NONE.toString());
running = hdfsCluster.getFileSystem()
- .getClient().isStoragePolicySatisfierRunning();
+ .getClient().isInternalSatisfierRunning();
Assert.assertFalse("SPS should stopped as configured.", running);
// Simulate the case by creating MOVER_ID file
@@ -576,7 +571,7 @@ public class TestStoragePolicySatisfier {
StoragePolicySatisfierMode.INTERNAL.toString());
running = hdfsCluster.getFileSystem()
- .getClient().isStoragePolicySatisfierRunning();
+ .getClient().isInternalSatisfierRunning();
Assert.assertFalse("SPS should not be able to run as file "
+ HdfsServerConstants.MOVER_ID_PATH + " is being hold.", running);
@@ -591,7 +586,7 @@ public class TestStoragePolicySatisfier {
DFSConfigKeys.DFS_STORAGE_POLICY_SATISFIER_MODE_KEY,
StoragePolicySatisfierMode.INTERNAL.toString());
running = hdfsCluster.getFileSystem()
- .getClient().isStoragePolicySatisfierRunning();
+ .getClient().isInternalSatisfierRunning();
Assert.assertTrue("SPS should be running as "
+ "Mover already exited", running);
@@ -623,7 +618,7 @@ public class TestStoragePolicySatisfier {
HdfsServerConstants.MOVER_ID_PATH, 0, (short) 1, 0);
restartNamenode();
boolean running = hdfsCluster.getFileSystem()
- .getClient().isStoragePolicySatisfierRunning();
+ .getClient().isInternalSatisfierRunning();
Assert.assertTrue("SPS should be running as "
+ "no Mover really running", running);
} finally {
@@ -1293,8 +1288,8 @@ public class TestStoragePolicySatisfier {
sps.getStorageMovementQueue().activate();
INode rootINode = fsDir.getINode("/root");
- hdfsCluster.getNamesystem().getBlockManager()
- .addSPSPathId(rootINode.getId());
+ hdfsCluster.getNamesystem().getBlockManager().getSPSManager()
+ .addPathId(rootINode.getId());
//Wait for thread to reach U.
Thread.sleep(1000);
@@ -1360,8 +1355,8 @@ public class TestStoragePolicySatisfier {
sps.getStorageMovementQueue().activate();
INode rootINode = fsDir.getINode("/root");
- hdfsCluster.getNamesystem().getBlockManager()
- .addSPSPathId(rootINode.getId());
+ hdfsCluster.getNamesystem().getBlockManager().getSPSManager()
+ .addPathId(rootINode.getId());
// Wait for thread to reach U.
Thread.sleep(1000);
@@ -1704,7 +1699,8 @@ public class TestStoragePolicySatisfier {
private void waitForAttemptedItems(long expectedBlkMovAttemptedCount,
int timeout) throws TimeoutException, InterruptedException {
BlockManager blockManager = hdfsCluster.getNamesystem().getBlockManager();
- final StoragePolicySatisfier sps = blockManager.getStoragePolicySatisfier();
+ final StoragePolicySatisfier sps = (StoragePolicySatisfier) blockManager
+ .getSPSManager().getInternalSPSService();
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
@@ -1723,7 +1719,8 @@ public class TestStoragePolicySatisfier {
long expectedMovementFinishedBlocksCount, int timeout)
throws TimeoutException, InterruptedException {
BlockManager blockManager = hdfsCluster.getNamesystem().getBlockManager();
- final StoragePolicySatisfier sps = blockManager.getStoragePolicySatisfier();
+ final StoragePolicySatisfier sps = (StoragePolicySatisfier) blockManager
+ .getSPSManager().getInternalSPSService();
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5262e30/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java
index 2257608..ef12300 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/sps/TestStoragePolicySatisfierWithStripedFile.java
@@ -495,7 +495,8 @@ public class TestStoragePolicySatisfierWithStripedFile {
long expectedBlkMovAttemptedCount, int timeout)
throws TimeoutException, InterruptedException {
BlockManager blockManager = cluster.getNamesystem().getBlockManager();
- final StoragePolicySatisfier sps = blockManager.getStoragePolicySatisfier();
+ final StoragePolicySatisfier sps = (StoragePolicySatisfier) blockManager
+ .getSPSManager().getInternalSPSService();
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
@@ -566,7 +567,8 @@ public class TestStoragePolicySatisfierWithStripedFile {
long expectedMoveFinishedBlks, int timeout)
throws TimeoutException, InterruptedException {
BlockManager blockManager = cluster.getNamesystem().getBlockManager();
- final StoragePolicySatisfier sps = blockManager.getStoragePolicySatisfier();
+ final StoragePolicySatisfier sps = (StoragePolicySatisfier) blockManager
+ .getSPSManager().getInternalSPSService();
Assert.assertNotNull("Failed to get SPS object reference!", sps);
GenericTestUtils.waitFor(new Supplier<Boolean>() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5262e30/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
index 15a4271..0546f39 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
@@ -133,7 +133,7 @@ public class TestExternalStoragePolicySatisfier
BlockManager blkMgr = cluster.getNameNode().getNamesystem()
.getBlockManager();
- SPSService spsService = blkMgr.getSPSService();
+ SPSService spsService = blkMgr.getSPSManager().getInternalSPSService();
spsService.stopGracefully();
ExternalSPSContext context = new ExternalSPSContext(spsService,
@@ -143,12 +143,12 @@ public class TestExternalStoragePolicySatisfier
new ExternalBlockMovementListener();
ExternalSPSBlockMoveTaskHandler externalHandler =
new ExternalSPSBlockMoveTaskHandler(conf, nnc,
- blkMgr.getSPSService());
+ blkMgr.getSPSManager().getInternalSPSService());
externalHandler.init();
spsService.init(context,
- new ExternalSPSFileIDCollector(context, blkMgr.getSPSService()),
- externalHandler,
- blkMoveListener);
+ new ExternalSPSFileIDCollector(context,
+ blkMgr.getSPSManager().getInternalSPSService()),
+ externalHandler, blkMoveListener);
spsService.start(true, StoragePolicySatisfierMode.EXTERNAL);
return cluster;
}
@@ -156,14 +156,14 @@ public class TestExternalStoragePolicySatisfier
public void restartNamenode() throws IOException{
BlockManager blkMgr = getCluster().getNameNode().getNamesystem()
.getBlockManager();
- SPSService spsService = blkMgr.getSPSService();
+ SPSService spsService = blkMgr.getSPSManager().getInternalSPSService();
spsService.stopGracefully();
getCluster().restartNameNodes();
getCluster().waitActive();
blkMgr = getCluster().getNameNode().getNamesystem()
.getBlockManager();
- spsService = blkMgr.getSPSService();
+ spsService = blkMgr.getSPSManager().getInternalSPSService();
spsService.stopGracefully();
ExternalSPSContext context = new ExternalSPSContext(spsService,
@@ -172,12 +172,12 @@ public class TestExternalStoragePolicySatisfier
new ExternalBlockMovementListener();
ExternalSPSBlockMoveTaskHandler externalHandler =
new ExternalSPSBlockMoveTaskHandler(getConf(), nnc,
- blkMgr.getSPSService());
+ blkMgr.getSPSManager().getInternalSPSService());
externalHandler.init();
spsService.init(context,
- new ExternalSPSFileIDCollector(context, blkMgr.getSPSService()),
- externalHandler,
- blkMoveListener);
+ new ExternalSPSFileIDCollector(context,
+ blkMgr.getSPSManager().getInternalSPSService()),
+ externalHandler, blkMoveListener);
spsService.start(true, StoragePolicySatisfierMode.EXTERNAL);
}
@@ -323,7 +323,7 @@ public class TestExternalStoragePolicySatisfier
DistributedFileSystem fs = getFS();
BlockManager blkMgr = getCluster().getNameNode().getNamesystem()
.getBlockManager();
- SPSService spsService = blkMgr.getSPSService();
+ SPSService spsService = blkMgr.getSPSManager().getInternalSPSService();
spsService.stopGracefully(); // stops SPS
// Creates 4 more files. Send all of them for satisfying the storage
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org