You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ar...@apache.org on 2013/11/22 21:51:08 UTC
svn commit: r1544672 [2/2] - in
/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project:
hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/
hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/ hadoop-hdfs/
hadoop-hdfs/src/main/java/ hado...
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js Fri Nov 22 20:51:06 2013
@@ -1,8 +1,9 @@
-(function(k){function n(b){b=b.f();return"object"===typeof b&&!0===b.h}function p(b,c){return"function"===typeof c?c.toString():c}function l(b,c,d,a,e){a=a||{};var m=d.a,g,f,l=a.d||"";if("undefined"!==typeof a.key)g=k.b.c(a.key,b,c);else if(n(c))g=c.f().i,c.f().g&&(e=function(){return!1});else return h.log("No key specified for filter in:"+l+" helper "),b;f=k.b.c(a.value,b,c);if(e(q(f,a.type,c),q(g,a.type,c))){n(c)&&(c.f().g=!0);if(m)return b.e(m,c);h.log("Missing body block in the "+l+" helper ")}else if(d["else"])return b.e(d["else"],
-c);return b}function q(b,c,d){if(b)switch(c||typeof b){case "number":return+b;case "string":return String(b);case "boolean":return Boolean("false"===b?!1:b);case "date":return new Date(b);case "context":return d.get(b)}return b}var h="undefined"!==typeof console?console:{log:function(){}};k.b={tap:function(b,c,d){var a=b;"function"===typeof b&&(!0===b.l?a=b():(a="",c.c(function(b){a+=b;return""}).e(b,d).p(),""===a&&(a=!1)));return a},sep:function(b,c,d){return c.stack.index===c.stack.m-1?b:d.a?d.a(b,
-c):b},idx:function(b,c,d){return d.a?d.a(b,c.push(c.stack.index)):b},contextDump:function(b,c,d,a){a=a||{};d=a.o||"output";a=a.key||"current";d=k.b.c(d,b,c);a=k.b.c(a,b,c);c="full"===a?JSON.stringify(c.stack,p,2):JSON.stringify(c.stack.head,p,2);return"console"===d?(h.log(c),b):b.write(c)},"if":function(b,c,d,a){var e=d.a,m=d["else"];if(a&&a.j){a=a.j;a=k.b.c(a,b,c);if(eval(a)){if(e)return b.e(d.a,c);h.log("Missing body block in the if helper!");return b}if(m)return b.e(d["else"],c)}else h.log("No condition given in the if helper!");
-return b},math:function(b,c,d,a){if(a&&"undefined"!==typeof a.key&&a.method){var e=a.key,m=a.method,g=a.n;a=a.round;var f=null,e=k.b.c(e,b,c),g=k.b.c(g,b,c);switch(m){case "mod":0!==g&&-0!==g||h.log("operand for divide operation is 0/-0: expect Nan!");f=parseFloat(e)%parseFloat(g);break;case "add":f=parseFloat(e)+parseFloat(g);break;case "subtract":f=parseFloat(e)-parseFloat(g);break;case "multiply":f=parseFloat(e)*parseFloat(g);break;case "divide":0!==g&&-0!==g||h.log("operand for divide operation is 0/-0: expect Nan/Infinity!");
-f=parseFloat(e)/parseFloat(g);break;case "ceil":f=Math.ceil(parseFloat(e));break;case "floor":f=Math.floor(parseFloat(e));break;case "round":f=Math.round(parseFloat(e));break;case "abs":f=Math.abs(parseFloat(e));break;default:h.log("method passed is not supported")}if(null!==f)return a&&(f=Math.round(f)),d&&d.a?b.e(d.a,c.push({h:!0,g:!1,i:f})):b.write(f)}else h.log("Key is a required parameter for math helper along with method/operand!");return b},select:function(b,c,d,a){var e=d.a;if(a&&"undefined"!==
-typeof a.key){a=k.b.c(a.key,b,c);if(e)return b.e(d.a,c.push({h:!0,g:!1,i:a}));h.log("Missing body block in the select helper ")}else h.log("No key given in the select helper!");return b},eq:function(b,c,d,a){a&&(a.d="eq");return l(b,c,d,a,function(a,b){return b===a})},ne:function(b,c,d,a){return a?(a.d="ne",l(b,c,d,a,function(a,b){return b!==a})):b},lt:function(b,c,d,a){if(a)return a.d="lt",l(b,c,d,a,function(a,b){return b<a})},lte:function(b,c,d,a){return a?(a.d="lte",l(b,c,d,a,function(a,b){return b<=
-a})):b},gt:function(b,c,d,a){return a?(a.d="gt",l(b,c,d,a,function(a,b){return b>a})):b},gte:function(b,c,d,a){return a?(a.d="gte",l(b,c,d,a,function(a,b){return b>=a})):b},"default":function(b,c,d,a){a&&(a.d="default");return l(b,c,d,a,function(){return!0})},size:function(b,c,d,a){c=0;var e;a=a||{};if((a=a.key)&&!0!==a)if(k.isArray(a))c=a.length;else if(!isNaN(parseFloat(a))&&isFinite(a))c=a;else if("object"===typeof a)for(e in c=0,a)Object.hasOwnProperty.call(a,e)&&c++;else c=(a+"").length;else c=
-0;return b.write(c)}}})("undefined"!==typeof exports?module.k=require("dustjs-linkedin"):dust);
+(function(k){function n(b){b=b.current();return"object"===typeof b&&!0===b.isSelect}function p(b,c){return"function"===typeof c?c.toString():c}function l(b,c,d,a,e){a=a||{};var m=d.block,g,f,l=a.filterOpType||"";if("undefined"!==typeof a.key)g=k.helpers.tap(a.key,b,c);else if(n(c))g=c.current().selectKey,c.current().isResolved&&(e=function(){return!1});else return h.log("No key specified for filter in:"+l+" helper "),b;f=k.helpers.tap(a.value,b,c);if(e(q(f,a.type,c),q(g,a.type,c))){n(c)&&(c.current().isResolved=
+!0);if(m)return b.render(m,c);h.log("Missing body block in the "+l+" helper ")}else if(d["else"])return b.render(d["else"],c);return b}function q(b,c,d){if(b)switch(c||typeof b){case "number":return+b;case "string":return String(b);case "boolean":return Boolean("false"===b?!1:b);case "date":return new Date(b);case "context":return d.get(b)}return b}var h="undefined"!==typeof console?console:{log:function(){}};k.helpers={tap:function(b,c,d){var a=b;"function"===typeof b&&(!0===b.isFunction?a=b():(a=
+"",c.tap(function(b){a+=b;return""}).render(b,d).untap(),""===a&&(a=!1)));return a},sep:function(b,c,d){return c.stack.index===c.stack.of-1?b:d.block?d.block(b,c):b},idx:function(b,c,d){return d.block?d.block(b,c.push(c.stack.index)):b},contextDump:function(b,c,d,a){a=a||{};d=a.to||"output";a=a.key||"current";d=k.helpers.tap(d,b,c);a=k.helpers.tap(a,b,c);c="full"===a?JSON.stringify(c.stack,p,2):JSON.stringify(c.stack.head,p,2);return"console"===d?(h.log(c),b):b.write(c)},"if":function(b,c,d,a){var e=
+d.block,m=d["else"];if(a&&a.cond){a=a.cond;a=k.helpers.tap(a,b,c);if(eval(a)){if(e)return b.render(d.block,c);h.log("Missing body block in the if helper!");return b}if(m)return b.render(d["else"],c)}else h.log("No condition given in the if helper!");return b},math:function(b,c,d,a){if(a&&"undefined"!==typeof a.key&&a.method){var e=a.key,m=a.method,g=a.operand;a=a.round;var f=null,e=k.helpers.tap(e,b,c),g=k.helpers.tap(g,b,c);switch(m){case "mod":0!==g&&-0!==g||h.log("operand for divide operation is 0/-0: expect Nan!");
+f=parseFloat(e)%parseFloat(g);break;case "add":f=parseFloat(e)+parseFloat(g);break;case "subtract":f=parseFloat(e)-parseFloat(g);break;case "multiply":f=parseFloat(e)*parseFloat(g);break;case "divide":0!==g&&-0!==g||h.log("operand for divide operation is 0/-0: expect Nan/Infinity!");f=parseFloat(e)/parseFloat(g);break;case "ceil":f=Math.ceil(parseFloat(e));break;case "floor":f=Math.floor(parseFloat(e));break;case "round":f=Math.round(parseFloat(e));break;case "abs":f=Math.abs(parseFloat(e));break;
+default:h.log("method passed is not supported")}if(null!==f)return a&&(f=Math.round(f)),d&&d.block?b.render(d.block,c.push({isSelect:!0,isResolved:!1,selectKey:f})):b.write(f)}else h.log("Key is a required parameter for math helper along with method/operand!");return b},select:function(b,c,d,a){var e=d.block;if(a&&"undefined"!==typeof a.key){a=k.helpers.tap(a.key,b,c);if(e)return b.render(d.block,c.push({isSelect:!0,isResolved:!1,selectKey:a}));h.log("Missing body block in the select helper ")}else h.log("No key given in the select helper!");
+return b},eq:function(b,c,d,a){a&&(a.filterOpType="eq");return l(b,c,d,a,function(a,b){return b===a})},ne:function(b,c,d,a){return a?(a.filterOpType="ne",l(b,c,d,a,function(a,b){return b!==a})):b},lt:function(b,c,d,a){if(a)return a.filterOpType="lt",l(b,c,d,a,function(a,b){return b<a})},lte:function(b,c,d,a){return a?(a.filterOpType="lte",l(b,c,d,a,function(a,b){return b<=a})):b},gt:function(b,c,d,a){return a?(a.filterOpType="gt",l(b,c,d,a,function(a,b){return b>a})):b},gte:function(b,c,d,a){return a?
+(a.filterOpType="gte",l(b,c,d,a,function(a,b){return b>=a})):b},"default":function(b,c,d,a){a&&(a.filterOpType="default");return l(b,c,d,a,function(a,b){return!0})},size:function(b,c,d,a){c=0;var e;a=a||{};if((a=a.key)&&!0!==a)if(k.isArray(a))c=a.length;else if(!isNaN(parseFloat(a))&&isFinite(a))c=a;else if("object"===typeof a)for(e in c=0,a)Object.hasOwnProperty.call(a,e)&&c++;else c=(a+"").length;else c=0;return b.write(c)}}})("undefined"!==typeof exports?module.exports=require("dustjs-linkedin"):
+dust);
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java Fri Nov 22 20:51:06 2013
@@ -135,4 +135,25 @@ public class TestFSOutputSummer {
cluster.shutdown();
}
}
+
+ @Test
+ public void TestDFSCheckSumType() throws Exception{
+ Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BYTES_PER_CHECKSUM);
+ conf.set(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, "NULL");
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(NUM_OF_DATANODES)
+ .build();
+ fileSys = cluster.getFileSystem();
+ try {
+ Path file = new Path("try.dat");
+ Random rand = new Random(seed);
+ rand.nextBytes(expected);
+ writeFile1(file);
+ } finally {
+ fileSys.close();
+ cluster.shutdown();
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Fri Nov 22 20:51:06 2013
@@ -1157,10 +1157,11 @@ public class TestReplicationPolicy {
assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
final BlockInfo info = new BlockInfo(block1, 1);
- final MutableBlockCollection mbc = mock(MutableBlockCollection.class);
+ final BlockCollection mbc = mock(BlockCollection.class);
when(mbc.getLastBlock()).thenReturn(info);
when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1);
when(mbc.getBlockReplication()).thenReturn((short)1);
+ when(mbc.isUnderConstruction()).thenReturn(true);
ContentSummary cs = mock(ContentSummary.class);
when(cs.getLength()).thenReturn((long)1);
when(mbc.computeContentSummary()).thenReturn(cs);
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java Fri Nov 22 20:51:06 2013
@@ -82,9 +82,10 @@ public class CreateEditsLog {
blocks[iB].setBlockId(currentBlockId++);
}
- INodeFileUnderConstruction inode = new INodeFileUnderConstruction(
- inodeId.nextValue(), null, replication, 0, blockSize, blocks, p, "",
- "", null);
+ final INodeFile inode = new INodeFile(inodeId.nextValue(), null,
+ p, 0L, 0L, blocks, replication, blockSize);
+ inode.toUnderConstruction("", "", null);
+
// Append path to filename with information about blockIDs
String path = "_" + iF + "_B" + blocks[0].getBlockId() +
"_to_B" + blocks[blocksPerFile-1].getBlockId() + "_";
@@ -96,9 +97,10 @@ public class CreateEditsLog {
dirInode = new INodeDirectory(inodeId.nextValue(), null, p, 0L);
editLog.logMkDir(currentDir, dirInode);
}
- editLog.logOpenFile(filePath,
- new INodeFileUnderConstruction(inodeId.nextValue(), p, replication,
- 0, blockSize, "", "", null), false);
+ INodeFile fileUc = new INodeFile(inodeId.nextValue(), null,
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
+ fileUc.toUnderConstruction("", "", null);
+ editLog.logOpenFile(filePath, fileUc, false);
editLog.logCloseFile(filePath, inode);
if (currentBlockId - bidAtSync >= 2000) { // sync every 2K blocks
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java Fri Nov 22 20:51:06 2013
@@ -614,6 +614,47 @@ public class TestCacheDirectives {
}, 500, 60000);
}
+ private static void waitForCachedStats(final DistributedFileSystem dfs,
+ final long targetFilesAffected, final long targetBytesNeeded,
+ final long targetBytesCached,
+ final CacheDirectiveInfo filter, final String infoString)
+ throws Exception {
+ LOG.info("Polling listDirectives{" +
+ ((filter == null) ? "ALL" : filter.toString()) +
+ " for " + targetFilesAffected + " targetFilesAffected, " +
+ targetBytesNeeded + " targetBytesNeeded, " +
+ targetBytesCached + " targetBytesCached");
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ RemoteIterator<CacheDirectiveEntry> iter = null;
+ CacheDirectiveEntry entry = null;
+ try {
+ iter = dfs.listCacheDirectives(filter);
+ entry = iter.next();
+ } catch (IOException e) {
+ fail("got IOException while calling " +
+ "listCacheDirectives: " + e.getMessage());
+ }
+ Assert.assertNotNull(entry);
+ CacheDirectiveStats stats = entry.getStats();
+ if ((targetFilesAffected == stats.getFilesAffected()) &&
+ (targetBytesNeeded == stats.getBytesNeeded()) &&
+ (targetBytesCached == stats.getBytesCached())) {
+ return true;
+ } else {
+ LOG.info(infoString + ": filesAffected: " +
+ stats.getFilesAffected() + "/" + targetFilesAffected +
+ ", bytesNeeded: " +
+ stats.getBytesNeeded() + "/" + targetBytesNeeded +
+ ", bytesCached: " +
+ stats.getBytesCached() + "/" + targetBytesCached);
+ return false;
+ }
+ }
+ }, 500, 60000);
+ }
+
private static void checkNumCachedReplicas(final DistributedFileSystem dfs,
final List<Path> paths, final int expectedBlocks,
final int expectedReplicas)
@@ -804,21 +845,12 @@ public class TestCacheDirectives {
waitForCachedBlocks(namenode, 4, 8,
"testWaitForCachedReplicasInDirectory:1");
// Verify that listDirectives gives the stats we want.
- RemoteIterator<CacheDirectiveEntry> iter =
- dfs.listCacheDirectives(new CacheDirectiveInfo.Builder().
- setPath(new Path("/foo")).
- build());
- CacheDirectiveEntry entry = iter.next();
- CacheDirectiveStats stats = entry.getStats();
- Assert.assertEquals(Long.valueOf(2),
- stats.getFilesAffected());
- Assert.assertEquals(Long.valueOf(
- 2 * numBlocksPerFile * BLOCK_SIZE * 2),
- stats.getBytesNeeded());
- Assert.assertEquals(Long.valueOf(
- 2 * numBlocksPerFile * BLOCK_SIZE * 2),
- stats.getBytesCached());
-
+ waitForCachedStats(dfs, 2,
+ 8 * BLOCK_SIZE, 8 * BLOCK_SIZE,
+ new CacheDirectiveInfo.Builder().
+ setPath(new Path("/foo")).
+ build(),
+ "testWaitForCachedReplicasInDirectory:2");
long id2 = dfs.addCacheDirective(
new CacheDirectiveInfo.Builder().
setPath(new Path("/foo/bar")).
@@ -827,44 +859,28 @@ public class TestCacheDirectives {
build());
// wait for an additional 2 cached replicas to come up
waitForCachedBlocks(namenode, 4, 10,
- "testWaitForCachedReplicasInDirectory:2");
+ "testWaitForCachedReplicasInDirectory:3");
// the directory directive's stats are unchanged
- iter = dfs.listCacheDirectives(
+ waitForCachedStats(dfs, 2,
+ 8 * BLOCK_SIZE, 8 * BLOCK_SIZE,
new CacheDirectiveInfo.Builder().
- setPath(new Path("/foo")).
- build());
- entry = iter.next();
- stats = entry.getStats();
- Assert.assertEquals(Long.valueOf(2),
- stats.getFilesAffected());
- Assert.assertEquals(Long.valueOf(
- 2 * numBlocksPerFile * BLOCK_SIZE * 2),
- stats.getBytesNeeded());
- Assert.assertEquals(Long.valueOf(
- 2 * numBlocksPerFile * BLOCK_SIZE * 2),
- stats.getBytesCached());
+ setPath(new Path("/foo")).
+ build(),
+ "testWaitForCachedReplicasInDirectory:4");
// verify /foo/bar's stats
- iter = dfs.listCacheDirectives(
+ waitForCachedStats(dfs, 1,
+ 4 * numBlocksPerFile * BLOCK_SIZE,
+ // only 3 because the file only has 3 replicas, not 4 as requested.
+ 3 * numBlocksPerFile * BLOCK_SIZE,
new CacheDirectiveInfo.Builder().
- setPath(new Path("/foo/bar")).
- build());
- entry = iter.next();
- stats = entry.getStats();
- Assert.assertEquals(Long.valueOf(1),
- stats.getFilesAffected());
- Assert.assertEquals(Long.valueOf(
- 4 * numBlocksPerFile * BLOCK_SIZE),
- stats.getBytesNeeded());
- // only 3 because the file only has 3 replicas, not 4 as requested.
- Assert.assertEquals(Long.valueOf(
- 3 * numBlocksPerFile * BLOCK_SIZE),
- stats.getBytesCached());
-
+ setPath(new Path("/foo/bar")).
+ build(),
+ "testWaitForCachedReplicasInDirectory:5");
// remove and watch numCached go to 0
dfs.removeCacheDirective(id);
dfs.removeCacheDirective(id2);
waitForCachedBlocks(namenode, 0, 0,
- "testWaitForCachedReplicasInDirectory:3");
+ "testWaitForCachedReplicasInDirectory:6");
} finally {
cluster.shutdown();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java Fri Nov 22 20:51:06 2013
@@ -43,8 +43,7 @@ public class TestCommitBlockSynchronizat
private static final long length = 200;
private static final long genStamp = 300;
- private FSNamesystem makeNameSystemSpy(Block block,
- INodeFileUnderConstruction file)
+ private FSNamesystem makeNameSystemSpy(Block block, INodeFile file)
throws IOException {
Configuration conf = new Configuration();
FSImage image = new FSImage(conf);
@@ -58,21 +57,26 @@ public class TestCommitBlockSynchronizat
blockInfo.setGenerationStamp(genStamp);
blockInfo.initializeBlockRecovery(genStamp);
doReturn(true).when(file).removeLastBlock(any(Block.class));
+ doReturn(true).when(file).isUnderConstruction();
doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class));
doReturn("").when(namesystemSpy).closeFileCommitBlocks(
- any(INodeFileUnderConstruction.class),
- any(BlockInfo.class));
+ any(INodeFile.class), any(BlockInfo.class));
doReturn("").when(namesystemSpy).persistBlocks(
- any(INodeFileUnderConstruction.class), anyBoolean());
+ any(INodeFile.class), anyBoolean());
doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog();
return namesystemSpy;
}
+ private INodeFile mockFileUnderConstruction() {
+ INodeFile file = mock(INodeFile.class);
+ return file;
+ }
+
@Test
public void testCommitBlockSynchronization() throws IOException {
- INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class);
+ INodeFile file = mockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = makeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
@@ -100,7 +104,7 @@ public class TestCommitBlockSynchronizat
@Test
public void testCommitBlockSynchronization2() throws IOException {
- INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class);
+ INodeFile file = mockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = makeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
@@ -124,7 +128,7 @@ public class TestCommitBlockSynchronizat
@Test
public void testCommitBlockSynchronizationWithDelete() throws IOException {
- INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class);
+ INodeFile file = mockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = makeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
@@ -144,7 +148,7 @@ public class TestCommitBlockSynchronizat
@Test
public void testCommitBlockSynchronizationWithClose() throws IOException {
- INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class);
+ INodeFile file = mockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = makeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[0];
@@ -171,7 +175,7 @@ public class TestCommitBlockSynchronizat
@Test
public void testCommitBlockSynchronizationWithCloseAndNonExistantTarget()
throws IOException {
- INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class);
+ INodeFile file = mockFileUnderConstruction();
Block block = new Block(blockId, length, genStamp);
FSNamesystem namesystemSpy = makeNameSystemSpy(block, file);
DatanodeID[] newTargets = new DatanodeID[]{
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Fri Nov 22 20:51:06 2013
@@ -63,6 +63,7 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
@@ -152,9 +153,10 @@ public class TestEditLog {
FSEditLog editLog = namesystem.getEditLog();
for (int i = 0; i < numTransactions; i++) {
- INodeFileUnderConstruction inode = new INodeFileUnderConstruction(
- namesystem.allocateNewInodeId(), p, replication, blockSize, 0, "",
- "", null);
+ INodeFile inode = new INodeFile(namesystem.allocateNewInodeId(), null,
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
+ inode.toUnderConstruction("", "", null);
+
editLog.logOpenFile("/filename" + (startIndex + i), inode, false);
editLog.logCloseFile("/filename" + (startIndex + i), inode);
editLog.logSync();
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java Fri Nov 22 20:51:06 2013
@@ -29,6 +29,8 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
+import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -59,6 +61,7 @@ import org.apache.hadoop.hdfs.protocol.L
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.util.Time;
import org.junit.Test;
import org.mockito.Mockito;
@@ -282,14 +285,6 @@ public class TestINodeFile {
assertTrue(fnfe.getMessage().contains("File does not exist"));
}
- //cast to INodeFileUnderConstruction, should fail
- try {
- INodeFileUnderConstruction.valueOf(from, path);
- fail();
- } catch(FileNotFoundException fnfe) {
- assertTrue(fnfe.getMessage().contains("File does not exist"));
- }
-
//cast to INodeDirectory, should fail
try {
INodeDirectory.valueOf(from, path);
@@ -306,14 +301,6 @@ public class TestINodeFile {
final INodeFile f = INodeFile.valueOf(from, path);
assertTrue(f == from);
- //cast to INodeFileUnderConstruction, should fail
- try {
- INodeFileUnderConstruction.valueOf(from, path);
- fail();
- } catch(IOException ioe) {
- assertTrue(ioe.getMessage().contains("File is not under construction"));
- }
-
//cast to INodeDirectory, should fail
try {
INodeDirectory.valueOf(from, path);
@@ -324,19 +311,14 @@ public class TestINodeFile {
}
{//cast from INodeFileUnderConstruction
- final INode from = new INodeFileUnderConstruction(
- INodeId.GRANDFATHER_INODE_ID, perm, replication, 0L, 0L, "client",
- "machine", null);
+ final INode from = new INodeFile(
+ INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, null, replication, 1024L);
+ from.asFile().toUnderConstruction("client", "machine", null);
//cast to INodeFile, should success
final INodeFile f = INodeFile.valueOf(from, path);
assertTrue(f == from);
- //cast to INodeFileUnderConstruction, should success
- final INodeFileUnderConstruction u = INodeFileUnderConstruction.valueOf(
- from, path);
- assertTrue(u == from);
-
//cast to INodeDirectory, should fail
try {
INodeDirectory.valueOf(from, path);
@@ -358,14 +340,6 @@ public class TestINodeFile {
assertTrue(fnfe.getMessage().contains("Path is not a file"));
}
- //cast to INodeFileUnderConstruction, should fail
- try {
- INodeFileUnderConstruction.valueOf(from, path);
- fail();
- } catch(FileNotFoundException fnfe) {
- assertTrue(fnfe.getMessage().contains("Path is not a file"));
- }
-
//cast to INodeDirectory, should success
final INodeDirectory d = INodeDirectory.valueOf(from, path);
assertTrue(d == from);
@@ -1015,4 +989,24 @@ public class TestINodeFile {
}
}
}
+
+ @Test
+ public void testFileUnderConstruction() {
+ replication = 3;
+ final INodeFile file = new INodeFile(INodeId.GRANDFATHER_INODE_ID, null,
+ perm, 0L, 0L, null, replication, 1024L);
+ assertFalse(file.isUnderConstruction());
+
+ final String clientName = "client";
+ final String clientMachine = "machine";
+ file.toUnderConstruction(clientName, clientMachine, null);
+ assertTrue(file.isUnderConstruction());
+ FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature();
+ assertEquals(clientName, uc.getClientName());
+ assertEquals(clientMachine, uc.getClientMachine());
+ Assert.assertNull(uc.getClientNode());
+
+ file.toCompleteFile(Time.now());
+ assertFalse(file.isUnderConstruction());
+ }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java Fri Nov 22 20:51:06 2013
@@ -66,7 +66,6 @@ import org.apache.hadoop.hdfs.protocol.C
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
-import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
@@ -720,8 +719,8 @@ public class TestRetryCacheWithHA {
@Override
boolean checkNamenodeBeforeReturn() throws Exception {
- INodeFileUnderConstruction fileNode = (INodeFileUnderConstruction) cluster
- .getNamesystem(0).getFSDirectory().getINode4Write(file).asFile();
+ INodeFile fileNode = cluster.getNamesystem(0).getFSDirectory()
+ .getINode4Write(file).asFile();
BlockInfoUnderConstruction blkUC =
(BlockInfoUnderConstruction) (fileNode.getBlocks())[1];
int datanodeNum = blkUC.getExpectedStorageLocations().length;
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java Fri Nov 22 20:51:06 2013
@@ -1227,8 +1227,9 @@ public class TestRenameWithSnapshots {
out.write(content);
fooRef = fsdir.getINode4Write(foo2.toString());
assertTrue(fooRef instanceof INodeReference.DstReference);
- INode fooNode = fooRef.asFile();
- assertTrue(fooNode instanceof INodeFileUnderConstructionWithSnapshot);
+ INodeFile fooNode = fooRef.asFile();
+ assertTrue(fooNode instanceof INodeFileWithSnapshot);
+ assertTrue(fooNode.isUnderConstruction());
} finally {
if (out != null) {
out.close();
@@ -1237,8 +1238,9 @@ public class TestRenameWithSnapshots {
fooRef = fsdir.getINode4Write(foo2.toString());
assertTrue(fooRef instanceof INodeReference.DstReference);
- INode fooNode = fooRef.asFile();
+ INodeFile fooNode = fooRef.asFile();
assertTrue(fooNode instanceof INodeFileWithSnapshot);
+ assertFalse(fooNode.isUnderConstruction());
restartClusterAndCheckImage(true);
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java?rev=1544672&r1=1544671&r2=1544672&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java Fri Nov 22 20:51:06 2013
@@ -314,7 +314,9 @@ public class TestSnapshotBlocksMap {
assertEquals(BLOCKSIZE, blks[0].getNumBytes());
}
- /** Make sure we delete 0-sized block when deleting an INodeFileUC */
+ /**
+ * Make sure we delete 0-sized block when deleting an under-construction file
+ */
@Test
public void testDeletionWithZeroSizeBlock2() throws Exception {
final Path foo = new Path("/foo");