You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cu...@apache.org on 2014/08/20 03:34:47 UTC
svn commit: r1619019 [10/11] - in
/hadoop/common/branches/YARN-1051/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs-httpfs/src/main/j...
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStorageStateRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStorageStateRecovery.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStorageStateRecovery.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStorageStateRecovery.java Wed Aug 20 01:34:29 2014
@@ -239,7 +239,7 @@ public class TestDFSStorageStateRecovery
assertTrue(new File(baseDirs[i],"previous").isDirectory());
assertEquals(
UpgradeUtilities.checksumContents(
- NAME_NODE, new File(baseDirs[i],"previous")),
+ NAME_NODE, new File(baseDirs[i],"previous"), false),
UpgradeUtilities.checksumMasterNameNodeContents());
}
}
@@ -259,7 +259,8 @@ public class TestDFSStorageStateRecovery
if (currentShouldExist) {
for (int i = 0; i < baseDirs.length; i++) {
assertEquals(
- UpgradeUtilities.checksumContents(DATA_NODE, new File(baseDirs[i],"current")),
+ UpgradeUtilities.checksumContents(DATA_NODE,
+ new File(baseDirs[i],"current"), false),
UpgradeUtilities.checksumMasterDataNodeContents());
}
}
@@ -267,7 +268,8 @@ public class TestDFSStorageStateRecovery
for (int i = 0; i < baseDirs.length; i++) {
assertTrue(new File(baseDirs[i],"previous").isDirectory());
assertEquals(
- UpgradeUtilities.checksumContents(DATA_NODE, new File(baseDirs[i],"previous")),
+ UpgradeUtilities.checksumContents(DATA_NODE,
+ new File(baseDirs[i],"previous"), false),
UpgradeUtilities.checksumMasterDataNodeContents());
}
}
@@ -290,8 +292,8 @@ public class TestDFSStorageStateRecovery
if (currentShouldExist) {
for (int i = 0; i < baseDirs.length; i++) {
File bpCurDir = new File(baseDirs[i], Storage.STORAGE_DIR_CURRENT);
- assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, bpCurDir),
- UpgradeUtilities.checksumMasterBlockPoolContents());
+ assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, bpCurDir,
+ false), UpgradeUtilities.checksumMasterBlockPoolContents());
}
}
if (previousShouldExist) {
@@ -299,8 +301,8 @@ public class TestDFSStorageStateRecovery
File bpPrevDir = new File(baseDirs[i], Storage.STORAGE_DIR_PREVIOUS);
assertTrue(bpPrevDir.isDirectory());
assertEquals(
- UpgradeUtilities.checksumContents(DATA_NODE, bpPrevDir),
- UpgradeUtilities.checksumMasterBlockPoolContents());
+ UpgradeUtilities.checksumContents(DATA_NODE, bpPrevDir,
+ false), UpgradeUtilities.checksumMasterBlockPoolContents());
}
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java Wed Aug 20 01:34:29 2014
@@ -100,7 +100,7 @@ public class TestDFSUpgrade {
File previous = new File(baseDir, "previous");
assertExists(previous);
- assertEquals(UpgradeUtilities.checksumContents(NAME_NODE, previous),
+ assertEquals(UpgradeUtilities.checksumContents(NAME_NODE, previous, false),
UpgradeUtilities.checksumMasterNameNodeContents());
}
}
@@ -114,23 +114,25 @@ public class TestDFSUpgrade {
void checkDataNode(String[] baseDirs, String bpid) throws IOException {
for (int i = 0; i < baseDirs.length; i++) {
File current = new File(baseDirs[i], "current/" + bpid + "/current");
- assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, current),
+ assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, current, false),
UpgradeUtilities.checksumMasterDataNodeContents());
// block files are placed under <sd>/current/<bpid>/current/finalized
File currentFinalized =
MiniDFSCluster.getFinalizedDir(new File(baseDirs[i]), bpid);
- assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, currentFinalized),
+ assertEquals(UpgradeUtilities.checksumContents(DATA_NODE,
+ currentFinalized, true),
UpgradeUtilities.checksumMasterBlockPoolFinalizedContents());
File previous = new File(baseDirs[i], "current/" + bpid + "/previous");
assertTrue(previous.isDirectory());
- assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, previous),
+ assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, previous, false),
UpgradeUtilities.checksumMasterDataNodeContents());
File previousFinalized =
new File(baseDirs[i], "current/" + bpid + "/previous"+"/finalized");
- assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, previousFinalized),
+ assertEquals(UpgradeUtilities.checksumContents(DATA_NODE,
+ previousFinalized, true),
UpgradeUtilities.checksumMasterBlockPoolFinalizedContents());
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java Wed Aug 20 01:34:29 2014
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
import java.io.BufferedReader;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
@@ -80,7 +81,7 @@ public class TestDFSUpgradeFromImage {
long checksum;
}
- private static final Configuration upgradeConf;
+ static final Configuration upgradeConf;
static {
upgradeConf = new HdfsConfiguration();
@@ -95,7 +96,7 @@ public class TestDFSUpgradeFromImage {
boolean printChecksum = false;
- private void unpackStorage(String tarFileName)
+ void unpackStorage(String tarFileName, String referenceName)
throws IOException {
String tarFile = System.getProperty("test.cache.data", "build/test/cache")
+ "/" + tarFileName;
@@ -110,7 +111,7 @@ public class TestDFSUpgradeFromImage {
BufferedReader reader = new BufferedReader(new FileReader(
System.getProperty("test.cache.data", "build/test/cache")
- + "/" + HADOOP_DFS_DIR_TXT));
+ + "/" + referenceName));
String line;
while ( (line = reader.readLine()) != null ) {
@@ -285,7 +286,7 @@ public class TestDFSUpgradeFromImage {
*/
@Test
public void testUpgradeFromRel22Image() throws IOException {
- unpackStorage(HADOOP22_IMAGE);
+ unpackStorage(HADOOP22_IMAGE, HADOOP_DFS_DIR_TXT);
upgradeAndVerify(new MiniDFSCluster.Builder(upgradeConf).
numDataNodes(4));
}
@@ -296,7 +297,7 @@ public class TestDFSUpgradeFromImage {
*/
@Test
public void testUpgradeFromCorruptRel22Image() throws IOException {
- unpackStorage(HADOOP22_IMAGE);
+ unpackStorage(HADOOP22_IMAGE, HADOOP_DFS_DIR_TXT);
// Overwrite the md5 stored in the VERSION files
File baseDir = new File(MiniDFSCluster.getBaseDirectory());
@@ -333,7 +334,7 @@ public class TestDFSUpgradeFromImage {
*/
@Test
public void testUpgradeFromRel1ReservedImage() throws Exception {
- unpackStorage(HADOOP1_RESERVED_IMAGE);
+ unpackStorage(HADOOP1_RESERVED_IMAGE, HADOOP_DFS_DIR_TXT);
MiniDFSCluster cluster = null;
// Try it once without setting the upgrade flag to ensure it fails
final Configuration conf = new Configuration();
@@ -403,7 +404,7 @@ public class TestDFSUpgradeFromImage {
*/
@Test
public void testUpgradeFromRel023ReservedImage() throws Exception {
- unpackStorage(HADOOP023_RESERVED_IMAGE);
+ unpackStorage(HADOOP023_RESERVED_IMAGE, HADOOP_DFS_DIR_TXT);
MiniDFSCluster cluster = null;
// Try it once without setting the upgrade flag to ensure it fails
final Configuration conf = new Configuration();
@@ -468,7 +469,7 @@ public class TestDFSUpgradeFromImage {
*/
@Test
public void testUpgradeFromRel2ReservedImage() throws Exception {
- unpackStorage(HADOOP2_RESERVED_IMAGE);
+ unpackStorage(HADOOP2_RESERVED_IMAGE, HADOOP_DFS_DIR_TXT);
MiniDFSCluster cluster = null;
// Try it once without setting the upgrade flag to ensure it fails
final Configuration conf = new Configuration();
@@ -572,7 +573,7 @@ public class TestDFSUpgradeFromImage {
} while (dirList.hasMore());
}
- private void upgradeAndVerify(MiniDFSCluster.Builder bld)
+ void upgradeAndVerify(MiniDFSCluster.Builder bld)
throws IOException {
MiniDFSCluster cluster = null;
try {
@@ -601,7 +602,7 @@ public class TestDFSUpgradeFromImage {
*/
@Test
public void testUpgradeFromRel1BBWImage() throws IOException {
- unpackStorage(HADOOP1_BBW_IMAGE);
+ unpackStorage(HADOOP1_BBW_IMAGE, HADOOP_DFS_DIR_TXT);
Configuration conf = new Configuration(upgradeConf);
conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
System.getProperty("test.build.data") + File.separator +
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Wed Aug 20 01:34:29 2014
@@ -30,8 +30,12 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -39,6 +43,7 @@ import static org.junit.Assert.assertTha
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
@@ -61,8 +66,12 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Shell;
+import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -792,4 +801,68 @@ public class TestDFSUtil {
}
}
}
+
+ @Test
+ public void testGetPassword() throws Exception {
+ File testDir = new File(System.getProperty("test.build.data",
+ "target/test-dir"));
+
+ Configuration conf = new Configuration();
+ final String ourUrl =
+ JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+
+ File file = new File(testDir, "test.jks");
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ char[] keypass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'};
+ char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};
+
+ // ensure that we get nulls when the key isn't there
+ assertEquals(null, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_KEYPASSWORD_KEY));
+ assertEquals(null, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY));
+ assertEquals(null, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY));
+
+ // create new aliases
+ try {
+ provider.createCredentialEntry(
+ DFS_SERVER_HTTPS_KEYPASSWORD_KEY, keypass);
+
+ provider.createCredentialEntry(
+ DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY, storepass);
+
+ provider.createCredentialEntry(
+ DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY, trustpass);
+
+ // write out so that it can be found in checks
+ provider.flush();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ // make sure we get back the right key directly from api
+ assertArrayEquals(keypass, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_KEYPASSWORD_KEY).getCredential());
+ assertArrayEquals(storepass, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY).getCredential());
+ assertArrayEquals(trustpass, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY).getCredential());
+
+ // use WebAppUtils as would be used by loadSslConfiguration
+ Assert.assertEquals("keypass",
+ DFSUtil.getPassword(conf, DFS_SERVER_HTTPS_KEYPASSWORD_KEY));
+ Assert.assertEquals("storepass",
+ DFSUtil.getPassword(conf, DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY));
+ Assert.assertEquals("trustpass",
+ DFSUtil.getPassword(conf, DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY));
+
+ // let's make sure that a password that doesn't exist returns null
+ Assert.assertEquals(null, DFSUtil.getPassword(conf,"invalid-alias"));
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeBlockScanner.java Wed Aug 20 01:34:29 2014
@@ -445,19 +445,14 @@ public class TestDatanodeBlockScanner {
@Test
public void testReplicaInfoParsing() throws Exception {
- testReplicaInfoParsingSingle(BASE_PATH, new int[0]);
- testReplicaInfoParsingSingle(BASE_PATH + "/subdir1", new int[]{1});
- testReplicaInfoParsingSingle(BASE_PATH + "/subdir43", new int[]{43});
- testReplicaInfoParsingSingle(BASE_PATH + "/subdir1/subdir2/subdir3", new int[]{1, 2, 3});
- testReplicaInfoParsingSingle(BASE_PATH + "/subdir1/subdir2/subdir43", new int[]{1, 2, 43});
- testReplicaInfoParsingSingle(BASE_PATH + "/subdir1/subdir23/subdir3", new int[]{1, 23, 3});
- testReplicaInfoParsingSingle(BASE_PATH + "/subdir13/subdir2/subdir3", new int[]{13, 2, 3});
+ testReplicaInfoParsingSingle(BASE_PATH);
+ testReplicaInfoParsingSingle(BASE_PATH + "/subdir1");
+ testReplicaInfoParsingSingle(BASE_PATH + "/subdir1/subdir2/subdir3");
}
- private static void testReplicaInfoParsingSingle(String subDirPath, int[] expectedSubDirs) {
+ private static void testReplicaInfoParsingSingle(String subDirPath) {
File testFile = new File(subDirPath);
- assertArrayEquals(expectedSubDirs, ReplicaInfo.parseSubDirs(testFile).subDirs);
- assertEquals(BASE_PATH, ReplicaInfo.parseSubDirs(testFile).baseDirPath);
+ assertEquals(BASE_PATH, ReplicaInfo.parseBaseDir(testFile).baseDirPath);
}
@Test
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java Wed Aug 20 01:34:29 2014
@@ -21,19 +21,26 @@ import static org.apache.hadoop.test.Met
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertEquals;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.junit.Test;
/**
* This test ensures the all types of data node report work correctly.
*/
public class TestDatanodeReport {
+ static final Log LOG = LogFactory.getLog(TestDatanodeReport.class);
final static private Configuration conf = new HdfsConfiguration();
final static private int NUM_OF_DATANODES = 4;
@@ -50,20 +57,18 @@ public class TestDatanodeReport {
try {
//wait until the cluster is up
cluster.waitActive();
-
- InetSocketAddress addr = new InetSocketAddress("localhost",
- cluster.getNameNodePort());
- DFSClient client = new DFSClient(addr, conf);
-
- assertEquals(client.datanodeReport(DatanodeReportType.ALL).length,
- NUM_OF_DATANODES);
- assertEquals(client.datanodeReport(DatanodeReportType.LIVE).length,
- NUM_OF_DATANODES);
- assertEquals(client.datanodeReport(DatanodeReportType.DEAD).length, 0);
+ final String bpid = cluster.getNamesystem().getBlockPoolId();
+ final List<DataNode> datanodes = cluster.getDataNodes();
+ final DFSClient client = cluster.getFileSystem().dfs;
+
+ assertReports(NUM_OF_DATANODES, DatanodeReportType.ALL, client, datanodes, bpid);
+ assertReports(NUM_OF_DATANODES, DatanodeReportType.LIVE, client, datanodes, bpid);
+ assertReports(0, DatanodeReportType.DEAD, client, datanodes, bpid);
// bring down one datanode
- ArrayList<DataNode> datanodes = cluster.getDataNodes();
- datanodes.remove(datanodes.size()-1).shutdown();
+ final DataNode last = datanodes.get(datanodes.size() - 1);
+ LOG.info("XXX shutdown datanode " + last.getDatanodeUuid());
+ last.shutdown();
DatanodeInfo[] nodeInfo = client.datanodeReport(DatanodeReportType.DEAD);
while (nodeInfo.length != 1) {
@@ -74,22 +79,59 @@ public class TestDatanodeReport {
nodeInfo = client.datanodeReport(DatanodeReportType.DEAD);
}
- assertEquals(client.datanodeReport(DatanodeReportType.LIVE).length,
- NUM_OF_DATANODES-1);
- assertEquals(client.datanodeReport(DatanodeReportType.ALL).length,
- NUM_OF_DATANODES);
+ assertReports(NUM_OF_DATANODES, DatanodeReportType.ALL, client, datanodes, null);
+ assertReports(NUM_OF_DATANODES - 1, DatanodeReportType.LIVE, client, datanodes, null);
+ assertReports(1, DatanodeReportType.DEAD, client, datanodes, null);
Thread.sleep(5000);
assertGauge("ExpiredHeartbeats", 1, getMetrics("FSNamesystem"));
- }finally {
+ } finally {
cluster.shutdown();
}
}
-
- public static void main(String[] args) throws Exception {
- new TestDatanodeReport().testDatanodeReport();
- }
-}
-
+ final static Comparator<StorageReport> CMP = new Comparator<StorageReport>() {
+ @Override
+ public int compare(StorageReport left, StorageReport right) {
+ return left.getStorage().getStorageID().compareTo(
+ right.getStorage().getStorageID());
+ }
+ };
+ static void assertReports(int numDatanodes, DatanodeReportType type,
+ DFSClient client, List<DataNode> datanodes, String bpid) throws IOException {
+ final DatanodeInfo[] infos = client.datanodeReport(type);
+ assertEquals(numDatanodes, infos.length);
+ final DatanodeStorageReport[] reports = client.getDatanodeStorageReport(type);
+ assertEquals(numDatanodes, reports.length);
+
+ for(int i = 0; i < infos.length; i++) {
+ assertEquals(infos[i], reports[i].getDatanodeInfo());
+
+ final DataNode d = findDatanode(infos[i].getDatanodeUuid(), datanodes);
+ if (bpid != null) {
+ //check storage
+ final StorageReport[] computed = reports[i].getStorageReports();
+ Arrays.sort(computed, CMP);
+ final StorageReport[] expected = d.getFSDataset().getStorageReports(bpid);
+ Arrays.sort(expected, CMP);
+
+ assertEquals(expected.length, computed.length);
+ for(int j = 0; j < expected.length; j++) {
+ assertEquals(expected[j].getStorage().getStorageID(),
+ computed[j].getStorage().getStorageID());
+ }
+ }
+ }
+ }
+
+ static DataNode findDatanode(String id, List<DataNode> datanodes) {
+ for(DataNode d : datanodes) {
+ if (d.getDatanodeUuid().equals(id)) {
+ return d;
+ }
+ }
+ throw new IllegalStateException("Datnode " + id + " not in datanode list: "
+ + datanodes);
+ }
+}
\ No newline at end of file
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java Wed Aug 20 01:34:29 2014
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.FileNotFoundException;
@@ -32,6 +33,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.HardLink;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -39,6 +41,7 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.junit.Assert;
import org.junit.Test;
@@ -169,6 +172,7 @@ public class TestFileAppend{
}
} finally {
+ client.close();
fs.close();
cluster.shutdown();
}
@@ -380,4 +384,57 @@ public class TestFileAppend{
}
}
+ /**
+ * Old replica of the block should not be accepted as valid for append/read
+ */
+ @Test
+ public void testFailedAppendBlockRejection() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.set("dfs.client.block.write.replace-datanode-on-failure.enable",
+ "false");
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3)
+ .build();
+ DistributedFileSystem fs = null;
+ try {
+ fs = cluster.getFileSystem();
+ Path path = new Path("/test");
+ FSDataOutputStream out = fs.create(path);
+ out.writeBytes("hello\n");
+ out.close();
+
+ // stop one datanode
+ DataNodeProperties dnProp = cluster.stopDataNode(0);
+ String dnAddress = dnProp.datanode.getXferAddress().toString();
+ if (dnAddress.startsWith("/")) {
+ dnAddress = dnAddress.substring(1);
+ }
+
+ // append again to bump genstamps
+ for (int i = 0; i < 2; i++) {
+ out = fs.append(path);
+ out.writeBytes("helloagain\n");
+ out.close();
+ }
+
+ // re-open and make the block state as underconstruction
+ out = fs.append(path);
+ cluster.restartDataNode(dnProp, true);
+ // wait till the block report comes
+ Thread.sleep(2000);
+ // check the block locations, this should not contain restarted datanode
+ BlockLocation[] locations = fs.getFileBlockLocations(path, 0,
+ Long.MAX_VALUE);
+ String[] names = locations[0].getNames();
+ for (String node : names) {
+ if (node.equals(dnAddress)) {
+ fail("Failed append should not be present in latest block locations.");
+ }
+ }
+ out.close();
+ } finally {
+ IOUtils.closeStream(fs);
+ cluster.shutdown();
+ }
+ }
+
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java Wed Aug 20 01:34:29 2014
@@ -27,6 +27,7 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -35,6 +36,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
@@ -137,13 +139,15 @@ public class TestFileCorruption {
final String bpid = cluster.getNamesystem().getBlockPoolId();
File storageDir = cluster.getInstanceStorageDir(0, 0);
File dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
+ assertTrue("Data directory does not exist", dataDir.exists());
ExtendedBlock blk = getBlock(bpid, dataDir);
if (blk == null) {
storageDir = cluster.getInstanceStorageDir(0, 1);
dataDir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
blk = getBlock(bpid, dataDir);
}
- assertFalse(blk==null);
+ assertFalse("Data directory does not contain any blocks or there was an "
+ + "IO error", blk==null);
// start a third datanode
cluster.startDataNodes(conf, 1, true, null, null);
@@ -174,33 +178,15 @@ public class TestFileCorruption {
}
- private ExtendedBlock getBlock(String bpid, File dataDir) {
- assertTrue("data directory does not exist", dataDir.exists());
- File[] blocks = dataDir.listFiles();
- assertTrue("Blocks do not exist in dataDir", (blocks != null) && (blocks.length > 0));
-
- int idx = 0;
- String blockFileName = null;
- for (; idx < blocks.length; idx++) {
- blockFileName = blocks[idx].getName();
- if (blockFileName.startsWith("blk_") && !blockFileName.endsWith(".meta")) {
- break;
- }
- }
- if (blockFileName == null) {
+ public static ExtendedBlock getBlock(String bpid, File dataDir) {
+ List<File> metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles(dataDir);
+ if (metadataFiles == null || metadataFiles.isEmpty()) {
return null;
}
- long blockId = Long.parseLong(blockFileName.substring("blk_".length()));
- long blockTimeStamp = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
- for (idx=0; idx < blocks.length; idx++) {
- String fileName = blocks[idx].getName();
- if (fileName.startsWith(blockFileName) && fileName.endsWith(".meta")) {
- int startIndex = blockFileName.length()+1;
- int endIndex = fileName.length() - ".meta".length();
- blockTimeStamp = Long.parseLong(fileName.substring(startIndex, endIndex));
- break;
- }
- }
- return new ExtendedBlock(bpid, blockId, blocks[idx].length(), blockTimeStamp);
+ File metadataFile = metadataFiles.get(0);
+ File blockFile = Block.metaToBlockFile(metadataFile);
+ return new ExtendedBlock(bpid, Block.getBlockId(blockFile.getName()),
+ blockFile.length(), Block.getGenerationStamp(metadataFile.getName()));
}
+
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java Wed Aug 20 01:34:29 2014
@@ -26,6 +26,7 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -36,6 +37,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
@@ -47,6 +49,8 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Before;
@@ -297,7 +301,8 @@ public class TestSafeMode {
* assert that they are either allowed or fail as expected.
*/
@Test
- public void testOperationsWhileInSafeMode() throws IOException {
+ public void testOperationsWhileInSafeMode() throws IOException,
+ InterruptedException {
final Path file1 = new Path("/file1");
assertFalse(dfs.setSafeMode(SafeModeAction.SAFEMODE_GET));
@@ -407,6 +412,22 @@ public class TestSafeMode {
fail("getAclStatus failed while in SM");
}
+ // Test access
+ UserGroupInformation ugiX = UserGroupInformation.createRemoteUser("userX");
+ FileSystem myfs = ugiX.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws IOException {
+ return FileSystem.get(conf);
+ }
+ });
+ myfs.access(file1, FsAction.READ);
+ try {
+ myfs.access(file1, FsAction.WRITE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
assertFalse("Could not leave SM",
dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE));
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/UpgradeUtilities.java Wed Aug 20 01:34:29 2014
@@ -158,21 +158,23 @@ public class UpgradeUtilities {
FileUtil.fullyDelete(new File(datanodeStorage,"in_use.lock"));
}
namenodeStorageChecksum = checksumContents(NAME_NODE,
- new File(namenodeStorage, "current"));
+ new File(namenodeStorage, "current"), false);
File dnCurDir = new File(datanodeStorage, "current");
- datanodeStorageChecksum = checksumContents(DATA_NODE, dnCurDir);
+ datanodeStorageChecksum = checksumContents(DATA_NODE, dnCurDir, false);
File bpCurDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir),
"current");
- blockPoolStorageChecksum = checksumContents(DATA_NODE, bpCurDir);
+ blockPoolStorageChecksum = checksumContents(DATA_NODE, bpCurDir, false);
File bpCurFinalizeDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir),
"current/"+DataStorage.STORAGE_DIR_FINALIZED);
- blockPoolFinalizedStorageChecksum = checksumContents(DATA_NODE, bpCurFinalizeDir);
+ blockPoolFinalizedStorageChecksum = checksumContents(DATA_NODE,
+ bpCurFinalizeDir, true);
File bpCurRbwDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir),
"current/"+DataStorage.STORAGE_DIR_RBW);
- blockPoolRbwStorageChecksum = checksumContents(DATA_NODE, bpCurRbwDir);
+ blockPoolRbwStorageChecksum = checksumContents(DATA_NODE, bpCurRbwDir,
+ false);
}
// Private helper method that writes a file to the given file system.
@@ -266,36 +268,47 @@ public class UpgradeUtilities {
/**
* Compute the checksum of all the files in the specified directory.
- * The contents of subdirectories are not included. This method provides
- * an easy way to ensure equality between the contents of two directories.
+ * This method provides an easy way to ensure equality between the contents
+ * of two directories.
*
* @param nodeType if DATA_NODE then any file named "VERSION" is ignored.
* This is because this file file is changed every time
* the Datanode is started.
- * @param dir must be a directory. Subdirectories are ignored.
+ * @param dir must be a directory
+ * @param recursive whether or not to consider subdirectories
*
* @throws IllegalArgumentException if specified directory is not a directory
* @throws IOException if an IOException occurs while reading the files
* @return the computed checksum value
*/
- public static long checksumContents(NodeType nodeType, File dir) throws IOException {
+ public static long checksumContents(NodeType nodeType, File dir,
+ boolean recursive) throws IOException {
+ CRC32 checksum = new CRC32();
+ checksumContentsHelper(nodeType, dir, checksum, recursive);
+ return checksum.getValue();
+ }
+
+ public static void checksumContentsHelper(NodeType nodeType, File dir,
+ CRC32 checksum, boolean recursive) throws IOException {
if (!dir.isDirectory()) {
throw new IllegalArgumentException(
- "Given argument is not a directory:" + dir);
+ "Given argument is not a directory:" + dir);
}
File[] list = dir.listFiles();
Arrays.sort(list);
- CRC32 checksum = new CRC32();
for (int i = 0; i < list.length; i++) {
if (!list[i].isFile()) {
+ if (recursive) {
+ checksumContentsHelper(nodeType, list[i], checksum, recursive);
+ }
continue;
}
// skip VERSION and dfsUsed file for DataNodes
- if (nodeType == DATA_NODE &&
- (list[i].getName().equals("VERSION") ||
- list[i].getName().equals("dfsUsed"))) {
- continue;
+ if (nodeType == DATA_NODE &&
+ (list[i].getName().equals("VERSION") ||
+ list[i].getName().equals("dfsUsed"))) {
+ continue;
}
FileInputStream fis = null;
@@ -312,7 +325,6 @@ public class UpgradeUtilities {
}
}
}
- return checksum.getValue();
}
/**
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java Wed Aug 20 01:34:29 2014
@@ -31,25 +31,25 @@ import org.apache.hadoop.fs.permission.A
import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeRegistrationProto;
-import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeStorageProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockWithLocationsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlocksWithLocationsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointSignatureProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExportedBlockKeysProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
@@ -67,9 +67,18 @@ import org.apache.hadoop.hdfs.server.com
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
-import org.apache.hadoop.hdfs.server.protocol.*;
+import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
+import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
+import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
@@ -175,8 +184,10 @@ public class TestPBHelper {
private static BlockWithLocations getBlockWithLocations(int bid) {
final String[] datanodeUuids = {"dn1", "dn2", "dn3"};
final String[] storageIDs = {"s1", "s2", "s3"};
+ final StorageType[] storageTypes = {
+ StorageType.DISK, StorageType.DISK, StorageType.DISK};
return new BlockWithLocations(new Block(bid, 0, 1),
- datanodeUuids, storageIDs);
+ datanodeUuids, storageIDs, storageTypes);
}
private void compare(BlockWithLocations locs1, BlockWithLocations locs2) {
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java Wed Aug 20 01:34:29 2014
@@ -18,17 +18,23 @@
package org.apache.hadoop.hdfs.server.balancer;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.File;
import java.io.IOException;
+import java.io.PrintWriter;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Random;
+import java.util.Set;
import java.util.concurrent.TimeoutException;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -48,6 +54,8 @@ import org.apache.hadoop.hdfs.protocol.E
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.balancer.Balancer.Cli;
+import org.apache.hadoop.hdfs.server.balancer.Balancer.Parameters;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
@@ -81,7 +89,7 @@ public class TestBalancer {
private static final Random r = new Random();
static {
- Balancer.setBlockMoveWaitTime(1000L) ;
+ Dispatcher.setBlockMoveWaitTime(1000L) ;
}
static void initConf(Configuration conf) {
@@ -255,6 +263,18 @@ public class TestBalancer {
}
}
}
+
+ /**
+ * Wait until balanced: each datanode gives utilization within
+ * BALANCE_ALLOWED_VARIANCE of average
+ * @throws IOException
+ * @throws TimeoutException
+ */
+ static void waitForBalancer(long totalUsedSpace, long totalCapacity,
+ ClientProtocol client, MiniDFSCluster cluster, Balancer.Parameters p)
+ throws IOException, TimeoutException {
+ waitForBalancer(totalUsedSpace, totalCapacity, client, cluster, p, 0);
+ }
/**
* Wait until balanced: each datanode gives utilization within
@@ -263,11 +283,17 @@ public class TestBalancer {
* @throws TimeoutException
*/
static void waitForBalancer(long totalUsedSpace, long totalCapacity,
- ClientProtocol client, MiniDFSCluster cluster)
- throws IOException, TimeoutException {
+ ClientProtocol client, MiniDFSCluster cluster, Balancer.Parameters p,
+ int expectedExcludedNodes) throws IOException, TimeoutException {
long timeout = TIMEOUT;
long failtime = (timeout <= 0L) ? Long.MAX_VALUE
: Time.now() + timeout;
+ if (!p.nodesToBeIncluded.isEmpty()) {
+ totalCapacity = p.nodesToBeIncluded.size() * CAPACITY;
+ }
+ if (!p.nodesToBeExcluded.isEmpty()) {
+ totalCapacity -= p.nodesToBeExcluded.size() * CAPACITY;
+ }
final double avgUtilization = ((double)totalUsedSpace) / totalCapacity;
boolean balanced;
do {
@@ -275,9 +301,20 @@ public class TestBalancer {
client.getDatanodeReport(DatanodeReportType.ALL);
assertEquals(datanodeReport.length, cluster.getDataNodes().size());
balanced = true;
+ int actualExcludedNodeCount = 0;
for (DatanodeInfo datanode : datanodeReport) {
double nodeUtilization = ((double)datanode.getDfsUsed())
/ datanode.getCapacity();
+ if (Dispatcher.Util.isExcluded(p.nodesToBeExcluded, datanode)) {
+ assertTrue(nodeUtilization == 0);
+ actualExcludedNodeCount++;
+ continue;
+ }
+ if (!Dispatcher.Util.isIncluded(p.nodesToBeIncluded, datanode)) {
+ assertTrue(nodeUtilization == 0);
+ actualExcludedNodeCount++;
+ continue;
+ }
if (Math.abs(avgUtilization - nodeUtilization) > BALANCE_ALLOWED_VARIANCE) {
balanced = false;
if (Time.now() > failtime) {
@@ -294,6 +331,7 @@ public class TestBalancer {
break;
}
}
+ assertEquals(expectedExcludedNodes,actualExcludedNodeCount);
} while (!balanced);
}
@@ -307,22 +345,118 @@ public class TestBalancer {
}
return b.append("]").toString();
}
- /** This test start a cluster with specified number of nodes,
+ /**
+ * Class which contains information about the
+ * new nodes to be added to the cluster for balancing.
+ */
+ static abstract class NewNodeInfo {
+
+ Set<String> nodesToBeExcluded = new HashSet<String>();
+ Set<String> nodesToBeIncluded = new HashSet<String>();
+
+ abstract String[] getNames();
+ abstract int getNumberofNewNodes();
+ abstract int getNumberofIncludeNodes();
+ abstract int getNumberofExcludeNodes();
+
+ public Set<String> getNodesToBeIncluded() {
+ return nodesToBeIncluded;
+ }
+ public Set<String> getNodesToBeExcluded() {
+ return nodesToBeExcluded;
+ }
+ }
+
+ /**
+ * The host names of new nodes are specified
+ */
+ static class HostNameBasedNodes extends NewNodeInfo {
+ String[] hostnames;
+
+ public HostNameBasedNodes(String[] hostnames,
+ Set<String> nodesToBeExcluded, Set<String> nodesToBeIncluded) {
+ this.hostnames = hostnames;
+ this.nodesToBeExcluded = nodesToBeExcluded;
+ this.nodesToBeIncluded = nodesToBeIncluded;
+ }
+
+ @Override
+ String[] getNames() {
+ return hostnames;
+ }
+ @Override
+ int getNumberofNewNodes() {
+ return hostnames.length;
+ }
+ @Override
+ int getNumberofIncludeNodes() {
+ return nodesToBeIncluded.size();
+ }
+ @Override
+ int getNumberofExcludeNodes() {
+ return nodesToBeExcluded.size();
+ }
+ }
+
+ /**
+ * The number of data nodes to be started are specified.
+ * The data nodes will have same host name, but different port numbers.
+ *
+ */
+ static class PortNumberBasedNodes extends NewNodeInfo {
+ int newNodes;
+ int excludeNodes;
+ int includeNodes;
+
+ public PortNumberBasedNodes(int newNodes, int excludeNodes, int includeNodes) {
+ this.newNodes = newNodes;
+ this.excludeNodes = excludeNodes;
+ this.includeNodes = includeNodes;
+ }
+
+ @Override
+ String[] getNames() {
+ return null;
+ }
+ @Override
+ int getNumberofNewNodes() {
+ return newNodes;
+ }
+ @Override
+ int getNumberofIncludeNodes() {
+ return includeNodes;
+ }
+ @Override
+ int getNumberofExcludeNodes() {
+ return excludeNodes;
+ }
+ }
+
+ private void doTest(Configuration conf, long[] capacities, String[] racks,
+ long newCapacity, String newRack, boolean useTool) throws Exception {
+ doTest(conf, capacities, racks, newCapacity, newRack, null, useTool, false);
+ }
+
+ /** This test start a cluster with specified number of nodes,
* and fills it to be 30% full (with a single file replicated identically
* to all datanodes);
* It then adds one new empty node and starts balancing.
- *
+ *
* @param conf - configuration
* @param capacities - array of capacities of original nodes in cluster
* @param racks - array of racks for original nodes in cluster
* @param newCapacity - new node's capacity
* @param newRack - new node's rack
+ * @param nodes - information about new nodes to be started.
* @param useTool - if true run test via Cli with command-line argument
* parsing, etc. Otherwise invoke balancer API directly.
+ * @param useFile - if true, the hosts to included or excluded will be stored in a
+ * file and then later read from the file.
* @throws Exception
*/
- private void doTest(Configuration conf, long[] capacities, String[] racks,
- long newCapacity, String newRack, boolean useTool) throws Exception {
+ private void doTest(Configuration conf, long[] capacities,
+ String[] racks, long newCapacity, String newRack, NewNodeInfo nodes,
+ boolean useTool, boolean useFile) throws Exception {
LOG.info("capacities = " + long2String(capacities));
LOG.info("racks = " + Arrays.asList(racks));
LOG.info("newCapacity= " + newCapacity);
@@ -346,17 +480,75 @@ public class TestBalancer {
long totalUsedSpace = totalCapacity*3/10;
createFile(cluster, filePath, totalUsedSpace / numOfDatanodes,
(short) numOfDatanodes, 0);
- // start up an empty node with the same capacity and on the same rack
- cluster.startDataNodes(conf, 1, true, null,
- new String[]{newRack}, new long[]{newCapacity});
- totalCapacity += newCapacity;
+ if (nodes == null) { // there is no specification of new nodes.
+ // start up an empty node with the same capacity and on the same rack
+ cluster.startDataNodes(conf, 1, true, null,
+ new String[]{newRack}, null,new long[]{newCapacity});
+ totalCapacity += newCapacity;
+ } else {
+ //if running a test with "include list", include original nodes as well
+ if (nodes.getNumberofIncludeNodes()>0) {
+ for (DataNode dn: cluster.getDataNodes())
+ nodes.getNodesToBeIncluded().add(dn.getDatanodeId().getHostName());
+ }
+ String[] newRacks = new String[nodes.getNumberofNewNodes()];
+ long[] newCapacities = new long[nodes.getNumberofNewNodes()];
+ for (int i=0; i < nodes.getNumberofNewNodes(); i++) {
+ newRacks[i] = newRack;
+ newCapacities[i] = newCapacity;
+ }
+ // if host names are specified for the new nodes to be created.
+ if (nodes.getNames() != null) {
+ cluster.startDataNodes(conf, nodes.getNumberofNewNodes(), true, null,
+ newRacks, nodes.getNames(), newCapacities);
+ totalCapacity += newCapacity*nodes.getNumberofNewNodes();
+ } else { // host names are not specified
+ cluster.startDataNodes(conf, nodes.getNumberofNewNodes(), true, null,
+ newRacks, null, newCapacities);
+ totalCapacity += newCapacity*nodes.getNumberofNewNodes();
+ //populate the include nodes
+ if (nodes.getNumberofIncludeNodes() > 0) {
+ int totalNodes = cluster.getDataNodes().size();
+ for (int i=0; i < nodes.getNumberofIncludeNodes(); i++) {
+ nodes.getNodesToBeIncluded().add (cluster.getDataNodes().get(
+ totalNodes-1-i).getDatanodeId().getXferAddr());
+ }
+ }
+ //polulate the exclude nodes
+ if (nodes.getNumberofExcludeNodes() > 0) {
+ int totalNodes = cluster.getDataNodes().size();
+ for (int i=0; i < nodes.getNumberofExcludeNodes(); i++) {
+ nodes.getNodesToBeExcluded().add (cluster.getDataNodes().get(
+ totalNodes-1-i).getDatanodeId().getXferAddr());
+ }
+ }
+ }
+ }
+ // run balancer and validate results
+ Balancer.Parameters p = Balancer.Parameters.DEFAULT;
+ if (nodes != null) {
+ p = new Balancer.Parameters(
+ Balancer.Parameters.DEFAULT.policy,
+ Balancer.Parameters.DEFAULT.threshold,
+ nodes.getNodesToBeExcluded(), nodes.getNodesToBeIncluded());
+ }
+
+ int expectedExcludedNodes = 0;
+ if (nodes != null) {
+ if (!nodes.getNodesToBeExcluded().isEmpty()) {
+ expectedExcludedNodes = nodes.getNodesToBeExcluded().size();
+ } else if (!nodes.getNodesToBeIncluded().isEmpty()) {
+ expectedExcludedNodes =
+ cluster.getDataNodes().size() - nodes.getNodesToBeIncluded().size();
+ }
+ }
// run balancer and validate results
if (useTool) {
- runBalancerCli(conf, totalUsedSpace, totalCapacity);
+ runBalancerCli(conf, totalUsedSpace, totalCapacity, p, useFile, expectedExcludedNodes);
} else {
- runBalancer(conf, totalUsedSpace, totalCapacity);
+ runBalancer(conf, totalUsedSpace, totalCapacity, p, expectedExcludedNodes);
}
} finally {
cluster.shutdown();
@@ -365,36 +557,86 @@ public class TestBalancer {
private void runBalancer(Configuration conf,
long totalUsedSpace, long totalCapacity) throws Exception {
+ runBalancer(conf, totalUsedSpace, totalCapacity, Balancer.Parameters.DEFAULT, 0);
+ }
+
+ private void runBalancer(Configuration conf,
+ long totalUsedSpace, long totalCapacity, Balancer.Parameters p,
+ int excludedNodes) throws Exception {
waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
// start rebalancing
Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, conf);
+ final int r = Balancer.run(namenodes, p, conf);
if (conf.getInt(DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY,
DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT) ==0) {
- assertEquals(Balancer.ReturnStatus.NO_MOVE_PROGRESS.code, r);
+ assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
return;
} else {
- assertEquals(Balancer.ReturnStatus.SUCCESS.code, r);
+ assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
}
waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
LOG.info("Rebalancing with default ctor.");
- waitForBalancer(totalUsedSpace, totalCapacity, client, cluster);
+ waitForBalancer(totalUsedSpace, totalCapacity, client, cluster, p, excludedNodes);
}
-
+
private void runBalancerCli(Configuration conf,
- long totalUsedSpace, long totalCapacity) throws Exception {
+ long totalUsedSpace, long totalCapacity,
+ Balancer.Parameters p, boolean useFile, int expectedExcludedNodes) throws Exception {
waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
+ List <String> args = new ArrayList<String>();
+ args.add("-policy");
+ args.add("datanode");
+
+ File excludeHostsFile = null;
+ if (!p.nodesToBeExcluded.isEmpty()) {
+ args.add("-exclude");
+ if (useFile) {
+ excludeHostsFile = new File ("exclude-hosts-file");
+ PrintWriter pw = new PrintWriter(excludeHostsFile);
+ for (String host: p.nodesToBeExcluded) {
+ pw.write( host + "\n");
+ }
+ pw.close();
+ args.add("-f");
+ args.add("exclude-hosts-file");
+ } else {
+ args.add(StringUtils.join(p.nodesToBeExcluded, ','));
+ }
+ }
+
+ File includeHostsFile = null;
+ if (!p.nodesToBeIncluded.isEmpty()) {
+ args.add("-include");
+ if (useFile) {
+ includeHostsFile = new File ("include-hosts-file");
+ PrintWriter pw = new PrintWriter(includeHostsFile);
+ for (String host: p.nodesToBeIncluded){
+ pw.write( host + "\n");
+ }
+ pw.close();
+ args.add("-f");
+ args.add("include-hosts-file");
+ } else {
+ args.add(StringUtils.join(p.nodesToBeIncluded, ','));
+ }
+ }
- final String[] args = { "-policy", "datanode" };
final Tool tool = new Cli();
tool.setConf(conf);
- final int r = tool.run(args); // start rebalancing
+ final int r = tool.run(args.toArray(new String[0])); // start rebalancing
assertEquals("Tools should exit 0 on success", 0, r);
waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
LOG.info("Rebalancing with default ctor.");
- waitForBalancer(totalUsedSpace, totalCapacity, client, cluster);
+ waitForBalancer(totalUsedSpace, totalCapacity, client, cluster, p, expectedExcludedNodes);
+
+ if (excludeHostsFile != null && excludeHostsFile.exists()) {
+ excludeHostsFile.delete();
+ }
+ if (includeHostsFile != null && includeHostsFile.exists()) {
+ includeHostsFile.delete();
+ }
}
/** one-node cluster test*/
@@ -416,6 +658,71 @@ public class TestBalancer {
oneNodeTest(conf, false);
}
+ /* we first start a cluster and fill the cluster up to a certain size.
+ * then redistribute blocks according the required distribution.
+ * Then we start an empty datanode.
+ * Afterwards a balancer is run to balance the cluster.
+ * A partially filled datanode is excluded during balancing.
+ * This triggers a situation where one of the block's location is unknown.
+ */
+ @Test(timeout=100000)
+ public void testUnknownDatanode() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ long distribution[] = new long[] {50*CAPACITY/100, 70*CAPACITY/100, 0*CAPACITY/100};
+ long capacities[] = new long[]{CAPACITY, CAPACITY, CAPACITY};
+ String racks[] = new String[] {RACK0, RACK1, RACK1};
+
+ int numDatanodes = distribution.length;
+ if (capacities.length != numDatanodes || racks.length != numDatanodes) {
+ throw new IllegalArgumentException("Array length is not the same");
+ }
+
+ // calculate total space that need to be filled
+ final long totalUsedSpace = sum(distribution);
+
+ // fill the cluster
+ ExtendedBlock[] blocks = generateBlocks(conf, totalUsedSpace,
+ (short) numDatanodes);
+
+ // redistribute blocks
+ Block[][] blocksDN = distributeBlocks(
+ blocks, (short)(numDatanodes-1), distribution);
+
+ // restart the cluster: do NOT format the cluster
+ conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, "0.0f");
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes)
+ .format(false)
+ .racks(racks)
+ .simulatedCapacities(capacities)
+ .build();
+ try {
+ cluster.waitActive();
+ client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
+ ClientProtocol.class).getProxy();
+
+ for(int i = 0; i < 3; i++) {
+ cluster.injectBlocks(i, Arrays.asList(blocksDN[i]), null);
+ }
+
+ cluster.startDataNodes(conf, 1, true, null,
+ new String[]{RACK0}, null,new long[]{CAPACITY});
+ cluster.triggerHeartbeats();
+
+ Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
+ Set<String> datanodes = new HashSet<String>();
+ datanodes.add(cluster.getDataNodes().get(0).getDatanodeId().getHostName());
+ Balancer.Parameters p = new Balancer.Parameters(
+ Balancer.Parameters.DEFAULT.policy,
+ Balancer.Parameters.DEFAULT.threshold,
+ datanodes, Balancer.Parameters.DEFAULT.nodesToBeIncluded);
+ final int r = Balancer.run(namenodes, p, conf);
+ assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
/**
* Test parse method in Balancer#Cli class with threshold value out of
* boundaries.
@@ -440,7 +747,7 @@ public class TestBalancer {
}
}
- /** Test a cluster with even distribution,
+ /** Test a cluster with even distribution,
* then a new empty node is added to the cluster*/
@Test(timeout=100000)
public void testBalancer0() throws Exception {
@@ -547,7 +854,35 @@ public class TestBalancer {
} catch (IllegalArgumentException e) {
}
- parameters = new String[] { "-threshold 1 -policy" };
+ parameters = new String[] {"-threshold", "1", "-policy"};
+ try {
+ Balancer.Cli.parse(parameters);
+ fail(reason);
+ } catch (IllegalArgumentException e) {
+
+ }
+ parameters = new String[] {"-threshold", "1", "-include"};
+ try {
+ Balancer.Cli.parse(parameters);
+ fail(reason);
+ } catch (IllegalArgumentException e) {
+
+ }
+ parameters = new String[] {"-threshold", "1", "-exclude"};
+ try {
+ Balancer.Cli.parse(parameters);
+ fail(reason);
+ } catch (IllegalArgumentException e) {
+
+ }
+ parameters = new String[] {"-include", "-f"};
+ try {
+ Balancer.Cli.parse(parameters);
+ fail(reason);
+ } catch (IllegalArgumentException e) {
+
+ }
+ parameters = new String[] {"-exclude", "-f"};
try {
Balancer.Cli.parse(parameters);
fail(reason);
@@ -555,6 +890,13 @@ public class TestBalancer {
}
+ parameters = new String[] {"-include", "testnode1", "-exclude", "testnode2"};
+ try {
+ Balancer.Cli.parse(parameters);
+ fail("IllegalArgumentException is expected when both -exclude and -include are specified");
+ } catch (IllegalArgumentException e) {
+
+ }
}
/**
@@ -570,6 +912,183 @@ public class TestBalancer {
}
/**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerWithExcludeList() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> excludeHosts = new HashSet<String>();
+ excludeHosts.add( "datanodeY");
+ excludeHosts.add( "datanodeZ");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ excludeHosts, Parameters.DEFAULT.nodesToBeIncluded), false, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerWithExcludeListWithPorts() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 2, 0), false, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithExcludeList() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> excludeHosts = new HashSet<String>();
+ excludeHosts.add( "datanodeY");
+ excludeHosts.add( "datanodeZ");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"}, excludeHosts,
+ Parameters.DEFAULT.nodesToBeIncluded), true, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithExcludeListWithPorts() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 2, 0), true, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list in a file
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithExcludeListInAFile() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> excludeHosts = new HashSet<String>();
+ excludeHosts.add( "datanodeY");
+ excludeHosts.add( "datanodeZ");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ excludeHosts, Parameters.DEFAULT.nodesToBeIncluded), true, true);
+ }
+
+ /**
+ * Test a cluster with even distribution,G
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the exclude list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithExcludeListWithPortsInAFile() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 2, 0), true, true);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerWithIncludeList() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> includeHosts = new HashSet<String>();
+ includeHosts.add( "datanodeY");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ Parameters.DEFAULT.nodesToBeExcluded, includeHosts), false, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerWithIncludeListWithPorts() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 0, 1), false, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithIncludeList() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> includeHosts = new HashSet<String>();
+ includeHosts.add( "datanodeY");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ Parameters.DEFAULT.nodesToBeExcluded, includeHosts), true, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithIncludeListWithPorts() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 0, 1), true, false);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithIncludeListInAFile() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ Set<String> includeHosts = new HashSet<String>();
+ includeHosts.add( "datanodeY");
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1}, CAPACITY, RACK2,
+ new HostNameBasedNodes(new String[] {"datanodeX", "datanodeY", "datanodeZ"},
+ Parameters.DEFAULT.nodesToBeExcluded, includeHosts), true, true);
+ }
+
+ /**
+ * Test a cluster with even distribution,
+ * then three nodes are added to the cluster,
+ * runs balancer with two of the nodes in the include list
+ */
+ @Test(timeout=100000)
+ public void testBalancerCliWithIncludeListWithPortsInAFile() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ doTest(conf, new long[]{CAPACITY, CAPACITY}, new String[]{RACK0, RACK1},
+ CAPACITY, RACK2, new PortNumberBasedNodes(3, 0, 1), true, true);
+ }
+
+ /**
* @param args
*/
public static void main(String[] args) throws Exception {
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java Wed Aug 20 01:34:29 2014
@@ -44,7 +44,7 @@ public class TestBalancerWithHANameNodes
ClientProtocol client;
static {
- Balancer.setBlockMoveWaitTime(1000L);
+ Dispatcher.setBlockMoveWaitTime(1000L);
}
/**
@@ -97,10 +97,10 @@ public class TestBalancerWithHANameNodes
Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
assertEquals(1, namenodes.size());
assertTrue(namenodes.contains(HATestUtil.getLogicalUri(cluster)));
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, conf);
- assertEquals(Balancer.ReturnStatus.SUCCESS.code, r);
+ final int r = Balancer.run(namenodes, Balancer.Parameters.DEFAULT, conf);
+ assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
TestBalancer.waitForBalancer(totalUsedSpace, totalCapacity, client,
- cluster);
+ cluster, Balancer.Parameters.DEFAULT);
} finally {
cluster.shutdown();
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java Wed Aug 20 01:34:29 2014
@@ -73,7 +73,7 @@ public class TestBalancerWithMultipleNam
private static final Random RANDOM = new Random();
static {
- Balancer.setBlockMoveWaitTime(1000L) ;
+ Dispatcher.setBlockMoveWaitTime(1000L) ;
}
/** Common objects used in various methods. */
@@ -159,8 +159,8 @@ public class TestBalancerWithMultipleNam
// start rebalancing
final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(s.conf);
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, s.conf);
- Assert.assertEquals(Balancer.ReturnStatus.SUCCESS.code, r);
+ final int r = Balancer.run(namenodes, Balancer.Parameters.DEFAULT, s.conf);
+ Assert.assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
LOG.info("BALANCER 2");
wait(s.clients, totalUsed, totalCapacity);
@@ -195,7 +195,7 @@ public class TestBalancerWithMultipleNam
balanced = true;
for(int d = 0; d < used.length; d++) {
final double p = used[d]*100.0/cap[d];
- balanced = p <= avg + Balancer.Parameters.DEFALUT.threshold;
+ balanced = p <= avg + Balancer.Parameters.DEFAULT.threshold;
if (!balanced) {
if (i % 100 == 0) {
LOG.warn("datanodes " + d + " is not yet balanced: "
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java Wed Aug 20 01:34:29 2014
@@ -75,7 +75,7 @@ public class TestBalancerWithNodeGroup {
static final int DEFAULT_BLOCK_SIZE = 100;
static {
- Balancer.setBlockMoveWaitTime(1000L) ;
+ Dispatcher.setBlockMoveWaitTime(1000L) ;
}
static Configuration createConf() {
@@ -175,8 +175,8 @@ public class TestBalancerWithNodeGroup {
// start rebalancing
Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, conf);
- assertEquals(Balancer.ReturnStatus.SUCCESS.code, r);
+ final int r = Balancer.run(namenodes, Balancer.Parameters.DEFAULT, conf);
+ assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
waitForHeartBeat(totalUsedSpace, totalCapacity);
LOG.info("Rebalancing with default factor.");
@@ -189,9 +189,9 @@ public class TestBalancerWithNodeGroup {
// start rebalancing
Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- final int r = Balancer.run(namenodes, Balancer.Parameters.DEFALUT, conf);
- Assert.assertTrue(r == Balancer.ReturnStatus.SUCCESS.code ||
- (r == Balancer.ReturnStatus.NO_MOVE_PROGRESS.code));
+ final int r = Balancer.run(namenodes, Balancer.Parameters.DEFAULT, conf);
+ Assert.assertTrue(r == ExitStatus.SUCCESS.getExitCode() ||
+ (r == ExitStatus.NO_MOVE_PROGRESS.getExitCode()));
waitForHeartBeat(totalUsedSpace, totalCapacity);
LOG.info("Rebalancing with default factor.");
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java Wed Aug 20 01:34:29 2014
@@ -268,4 +268,14 @@ public class BlockManagerTestUtil {
}
return reports.toArray(StorageReport.EMPTY_ARRAY);
}
+
+ /**
+ * Have DatanodeManager check decommission state.
+ * @param dm the DatanodeManager to manipulate
+ */
+ public static void checkDecommissionState(DatanodeManager dm,
+ DatanodeDescriptor node) {
+ dm.checkDecommissionState(node);
+ }
+
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java Wed Aug 20 01:34:29 2014
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -59,17 +60,24 @@ public class TestBlockInfo {
@Test
- public void testReplaceStorageIfDifferetnOneAlreadyExistedFromSameDataNode() throws Exception {
- BlockInfo blockInfo = new BlockInfo(3);
+ public void testReplaceStorage() throws Exception {
+ // Create two dummy storages.
final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1", "127.0.0.1");
final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2"));
+ final int NUM_BLOCKS = 10;
+ BlockInfo[] blockInfos = new BlockInfo[NUM_BLOCKS];
- blockInfo.addStorage(storage1);
- boolean added = blockInfo.addStorage(storage2);
+ // Create a few dummy blocks and add them to the first storage.
+ for (int i = 0; i < NUM_BLOCKS; ++i) {
+ blockInfos[i] = new BlockInfo(3);
+ storage1.addBlock(blockInfos[i]);
+ }
- Assert.assertFalse(added);
- Assert.assertEquals(storage2, blockInfo.getStorageInfo(0));
+ // Try to move one of the blocks to a different storage.
+ boolean added = storage2.addBlock(blockInfos[NUM_BLOCKS/2]);
+ Assert.assertThat(added, is(false));
+ Assert.assertThat(blockInfos[NUM_BLOCKS/2].getStorageInfo(0), is(storage2));
}
@Test
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java Wed Aug 20 01:34:29 2014
@@ -368,7 +368,7 @@ public class TestBlockManager {
DatanodeStorageInfo[] pipeline) throws IOException {
for (int i = 1; i < pipeline.length; i++) {
DatanodeStorageInfo storage = pipeline[i];
- bm.addBlock(storage.getDatanodeDescriptor(), storage.getStorageID(), blockInfo, null);
+ bm.addBlock(storage, blockInfo, null);
blockInfo.addStorage(storage);
}
}
@@ -549,12 +549,12 @@ public class TestBlockManager {
// send block report, should be processed
reset(node);
- bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool",
+ bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
new BlockListAsLongs(null, null));
assertEquals(1, ds.getBlockReportCount());
// send block report again, should NOT be processed
reset(node);
- bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool",
+ bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
new BlockListAsLongs(null, null));
assertEquals(1, ds.getBlockReportCount());
@@ -566,7 +566,7 @@ public class TestBlockManager {
assertEquals(0, ds.getBlockReportCount()); // ready for report again
// send block report, should be processed after restart
reset(node);
- bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool",
+ bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
new BlockListAsLongs(null, null));
assertEquals(1, ds.getBlockReportCount());
}
@@ -595,7 +595,7 @@ public class TestBlockManager {
// send block report while pretending to already have blocks
reset(node);
doReturn(1).when(node).numBlocks();
- bm.processReport(node, new DatanodeStorage(ds.getStorageID()), "pool",
+ bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
new BlockListAsLongs(null, null));
assertEquals(1, ds.getBlockReportCount());
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java Wed Aug 20 01:34:29 2014
@@ -33,6 +33,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.CorruptReplicasMap.Reason;
import org.junit.Test;
@@ -89,14 +90,14 @@ public class TestCorruptReplicaInfo {
DatanodeDescriptor dn1 = DFSTestUtil.getLocalDatanodeDescriptor();
DatanodeDescriptor dn2 = DFSTestUtil.getLocalDatanodeDescriptor();
- crm.addToCorruptReplicasMap(getBlock(0), dn1, "TEST");
+ addToCorruptReplicasMap(crm, getBlock(0), dn1);
assertEquals("Number of corrupt blocks not returning correctly",
1, crm.size());
- crm.addToCorruptReplicasMap(getBlock(1), dn1, "TEST");
+ addToCorruptReplicasMap(crm, getBlock(1), dn1);
assertEquals("Number of corrupt blocks not returning correctly",
2, crm.size());
- crm.addToCorruptReplicasMap(getBlock(1), dn2, "TEST");
+ addToCorruptReplicasMap(crm, getBlock(1), dn2);
assertEquals("Number of corrupt blocks not returning correctly",
2, crm.size());
@@ -109,7 +110,7 @@ public class TestCorruptReplicaInfo {
0, crm.size());
for (Long block_id: block_ids) {
- crm.addToCorruptReplicasMap(getBlock(block_id), dn1, "TEST");
+ addToCorruptReplicasMap(crm, getBlock(block_id), dn1);
}
assertEquals("Number of corrupt blocks not returning correctly",
@@ -127,4 +128,9 @@ public class TestCorruptReplicaInfo {
crm.getCorruptReplicaBlockIds(10, 7L)));
}
+
+ private static void addToCorruptReplicasMap(CorruptReplicasMap crm,
+ Block blk, DatanodeDescriptor dn) {
+ crm.addToCorruptReplicasMap(blk, dn, "TEST", Reason.NONE);
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java Wed Aug 20 01:34:29 2014
@@ -63,16 +63,16 @@ public class TestDatanodeDescriptor {
assertTrue(storages.length > 0);
final String storageID = storages[0].getStorageID();
// add first block
- assertTrue(dd.addBlock(storageID, blk));
+ assertTrue(storages[0].addBlock(blk));
assertEquals(1, dd.numBlocks());
// remove a non-existent block
assertFalse(dd.removeBlock(blk1));
assertEquals(1, dd.numBlocks());
// add an existent block
- assertFalse(dd.addBlock(storageID, blk));
+ assertFalse(storages[0].addBlock(blk));
assertEquals(1, dd.numBlocks());
// add second block
- assertTrue(dd.addBlock(storageID, blk1));
+ assertTrue(storages[0].addBlock(blk1));
assertEquals(2, dd.numBlocks());
// remove first block
assertTrue(dd.removeBlock(blk));