You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/10/01 03:25:41 UTC
[1/3] git commit: HDFS-7153. Add storagePolicy to NN edit log during
file creation. (Arpit Agarwal)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 dde2ed13c -> a45ad330f
HDFS-7153. Add storagePolicy to NN edit log during file creation. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e9e9cf7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e9e9cf7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e9e9cf7
Branch: refs/heads/HDFS-6581
Commit: 9e9e9cf71100fe7ceebd5a5ac27b164059b708c9
Parents: a469833
Author: arp <ar...@apache.org>
Authored: Tue Sep 30 18:10:13 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Tue Sep 30 18:10:13 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSDirectory.java | 17 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 3 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 2 +-
.../hdfs/server/namenode/FSEditLogOp.java | 16 ++
.../server/namenode/FSImageSerialization.java | 10 +
.../hdfs/server/namenode/FSNamesystem.java | 2 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 4999 -> 5065 bytes
.../src/test/resources/editsStored.xml | 212 +++++++++----------
9 files changed, 152 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e9e9cf7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 40891bf..393bb00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -537,6 +537,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6779. Add missing version subcommand for hdfs.
(Sasaki Toru via wheat9)
+ HDFS-7153. Add storagePolicy to NN edit log during file creation.
+ (Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e9e9cf7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index b5cf6c6..aab2183 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -278,8 +278,16 @@ public class FSDirectory implements Closeable {
private static INodeFile newINodeFile(long id, PermissionStatus permissions,
long mtime, long atime, short replication, long preferredBlockSize) {
+ return newINodeFile(id, permissions, mtime, atime, replication, preferredBlockSize,
+ (byte)0);
+ }
+
+ private static INodeFile newINodeFile(long id, PermissionStatus permissions,
+ long mtime, long atime, short replication, long preferredBlockSize,
+ byte storagePolicyId) {
return new INodeFile(id, null, permissions, mtime, atime,
- BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize, (byte)0);
+ BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize,
+ storagePolicyId);
}
/**
@@ -329,17 +337,18 @@ public class FSDirectory implements Closeable {
long preferredBlockSize,
boolean underConstruction,
String clientName,
- String clientMachine) {
+ String clientMachine,
+ byte storagePolicyId) {
final INodeFile newNode;
assert hasWriteLock();
if (underConstruction) {
newNode = newINodeFile(id, permissions, modificationTime,
- modificationTime, replication, preferredBlockSize);
+ modificationTime, replication, preferredBlockSize, storagePolicyId);
newNode.toUnderConstruction(clientName, clientMachine);
} else {
newNode = newINodeFile(id, permissions, modificationTime, atime,
- replication, preferredBlockSize);
+ replication, preferredBlockSize, storagePolicyId);
}
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e9e9cf7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index d0f2d0a..e952e7c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -719,7 +719,8 @@ public class FSEditLog implements LogsPurgeable {
.setClientName(newNode.getFileUnderConstructionFeature().getClientName())
.setClientMachine(
newNode.getFileUnderConstructionFeature().getClientMachine())
- .setOverwrite(overwrite);
+ .setOverwrite(overwrite)
+ .setStoragePolicyId(newNode.getStoragePolicyID());
AclFeature f = newNode.getAclFeature();
if (f != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e9e9cf7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index d13199e..1b2b4ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -366,7 +366,7 @@ public class FSEditLogLoader {
addCloseOp.xAttrs,
replication, addCloseOp.mtime, addCloseOp.atime,
addCloseOp.blockSize, true, addCloseOp.clientName,
- addCloseOp.clientMachine);
+ addCloseOp.clientMachine, addCloseOp.storagePolicyId);
fsNamesys.leaseManager.addLease(addCloseOp.clientName, path);
// add the op into retry cache if necessary
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e9e9cf7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 112c3c1..98ea7d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -103,6 +103,7 @@ import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEditLogProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrEditLogProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException;
import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
@@ -410,6 +411,7 @@ public abstract class FSEditLogOp {
String clientName;
String clientMachine;
boolean overwrite;
+ byte storagePolicyId;
private AddCloseOp(FSEditLogOpCodes opCode) {
super(opCode);
@@ -495,6 +497,11 @@ public abstract class FSEditLogOp {
return (T)this;
}
+ <T extends AddCloseOp> T setStoragePolicyId(byte storagePolicyId) {
+ this.storagePolicyId = storagePolicyId;
+ return (T)this;
+ }
+
@Override
public void writeFields(DataOutputStream out) throws IOException {
FSImageSerialization.writeLong(inodeId, out);
@@ -514,6 +521,7 @@ public abstract class FSEditLogOp {
FSImageSerialization.writeString(clientName,out);
FSImageSerialization.writeString(clientMachine,out);
FSImageSerialization.writeBoolean(overwrite, out);
+ FSImageSerialization.writeByte(storagePolicyId, out);
// write clientId and callId
writeRpcIds(rpcClientId, rpcCallId, out);
}
@@ -585,6 +593,12 @@ public abstract class FSEditLogOp {
} else {
this.overwrite = false;
}
+ if (NameNodeLayoutVersion.supports(
+ NameNodeLayoutVersion.Feature.BLOCK_STORAGE_POLICY, logVersion)) {
+ this.storagePolicyId = FSImageSerialization.readByte(in);
+ } else {
+ this.storagePolicyId = BlockStoragePolicySuite.ID_UNSPECIFIED;
+ }
// read clientId and callId
readRpcIds(in, logVersion);
} else {
@@ -645,6 +659,8 @@ public abstract class FSEditLogOp {
if (this.opCode == OP_ADD) {
appendRpcIdsToString(builder, rpcClientId, rpcCallId);
}
+ builder.append(", storagePolicyId=");
+ builder.append(storagePolicyId);
builder.append(", opCode=");
builder.append(opCode);
builder.append(", txid=");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e9e9cf7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index be70f4b..d3472fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -383,6 +383,12 @@ public class FSImageSerialization {
uBoolean.write(out);
}
+ /** write the byte value */
+ static void writeByte(byte value, DataOutputStream out)
+ throws IOException {
+ out.write(value);
+ }
+
/** read the int value */
static int readInt(DataInput in) throws IOException {
IntWritable uInt = TL_DATA.get().U_INT;
@@ -422,6 +428,10 @@ public class FSImageSerialization {
return bytes;
}
+ public static byte readByte(DataInput in) throws IOException {
+ return in.readByte();
+ }
+
/**
* Reading the path from the image and converting it to byte[][] directly
* this saves us an array copy and conversions to and from String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e9e9cf7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 9b4380c..a3d8dc7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2578,7 +2578,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
clientMachine, create, overwrite, createParent, replication,
blockSize, suite, protocolVersion, edek, logRetryCache);
stat = dir.getFileInfo(src, false,
- FSDirectory.isReservedRawName(srcArg), false);
+ FSDirectory.isReservedRawName(srcArg), true);
} catch (StandbyException se) {
skipSync = true;
throw se;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e9e9cf7/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index ecfbb9f..08607eb 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e9e9cf7/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index 8cafa9f..5e1d718 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -13,8 +13,8 @@
<TXID>2</TXID>
<DELEGATION_KEY>
<KEY_ID>1</KEY_ID>
- <EXPIRY_DATE>1410915997709</EXPIRY_DATE>
- <KEY>309e81e09dc6c75a</KEY>
+ <EXPIRY_DATE>1412805665311</EXPIRY_DATE>
+ <KEY>c1cad1109e33ae77</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -24,8 +24,8 @@
<TXID>3</TXID>
<DELEGATION_KEY>
<KEY_ID>2</KEY_ID>
- <EXPIRY_DATE>1410915997711</EXPIRY_DATE>
- <KEY>8a2399843e754bee</KEY>
+ <EXPIRY_DATE>1412805665314</EXPIRY_DATE>
+ <KEY>0632068587d6574c</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -37,19 +37,19 @@
<INODEID>16386</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798292</MTIME>
- <ATIME>1410224798292</ATIME>
+ <MTIME>1412114467969</MTIME>
+ <ATIME>1412114467969</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1374813776_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>5</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>13</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -60,14 +60,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798315</MTIME>
- <ATIME>1410224798292</ATIME>
+ <MTIME>1412114468019</MTIME>
+ <ATIME>1412114467969</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -88,9 +88,9 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1410224798322</TIMESTAMP>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>8</RPC_CALLID>
+ <TIMESTAMP>1412114468027</TIMESTAMP>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>16</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -99,9 +99,9 @@
<TXID>8</TXID>
<LENGTH>0</LENGTH>
<PATH>/file_moved</PATH>
- <TIMESTAMP>1410224798328</TIMESTAMP>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>9</RPC_CALLID>
+ <TIMESTAMP>1412114468034</TIMESTAMP>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>17</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -111,9 +111,9 @@
<LENGTH>0</LENGTH>
<INODEID>16387</INODEID>
<PATH>/directory_mkdir</PATH>
- <TIMESTAMP>1410224798335</TIMESTAMP>
+ <TIMESTAMP>1412114468041</TIMESTAMP>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>493</MODE>
</PERMISSION_STATUS>
@@ -146,8 +146,8 @@
<TXID>13</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot1</SNAPSHOTNAME>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>14</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>22</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -157,8 +157,8 @@
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME>
<SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>15</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>23</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -167,8 +167,8 @@
<TXID>15</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot2</SNAPSHOTNAME>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>16</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>24</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -179,19 +179,19 @@
<INODEID>16388</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798359</MTIME>
- <ATIME>1410224798359</ATIME>
+ <MTIME>1412114468073</MTIME>
+ <ATIME>1412114468073</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1374813776_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>17</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>25</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -202,14 +202,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798361</MTIME>
- <ATIME>1410224798359</ATIME>
+ <MTIME>1412114468075</MTIME>
+ <ATIME>1412114468073</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -265,10 +265,10 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1410224798379</TIMESTAMP>
+ <TIMESTAMP>1412114468093</TIMESTAMP>
<OPTIONS>NONE</OPTIONS>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>24</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>32</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -279,19 +279,19 @@
<INODEID>16389</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798382</MTIME>
- <ATIME>1410224798382</ATIME>
+ <MTIME>1412114468097</MTIME>
+ <ATIME>1412114468097</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1374813776_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>26</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>34</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -396,8 +396,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798476</MTIME>
- <ATIME>1410224798382</ATIME>
+ <MTIME>1412114468349</MTIME>
+ <ATIME>1412114468097</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -418,7 +418,7 @@
<GENSTAMP>1003</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -432,19 +432,19 @@
<INODEID>16390</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798479</MTIME>
- <ATIME>1410224798479</ATIME>
+ <MTIME>1412114468351</MTIME>
+ <ATIME>1412114468351</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1374813776_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>39</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>47</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -549,8 +549,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798501</MTIME>
- <ATIME>1410224798479</ATIME>
+ <MTIME>1412114468370</MTIME>
+ <ATIME>1412114468351</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -571,7 +571,7 @@
<GENSTAMP>1006</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -585,19 +585,19 @@
<INODEID>16391</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798504</MTIME>
- <ATIME>1410224798504</ATIME>
+ <MTIME>1412114468373</MTIME>
+ <ATIME>1412114468373</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1374813776_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>51</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>59</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -702,8 +702,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798530</MTIME>
- <ATIME>1410224798504</ATIME>
+ <MTIME>1412114468392</MTIME>
+ <ATIME>1412114468373</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -724,7 +724,7 @@
<GENSTAMP>1009</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -736,13 +736,13 @@
<TXID>57</TXID>
<LENGTH>0</LENGTH>
<TRG>/file_concat_target</TRG>
- <TIMESTAMP>1410224798533</TIMESTAMP>
+ <TIMESTAMP>1412114468395</TIMESTAMP>
<SOURCES>
<SOURCE1>/file_concat_0</SOURCE1>
<SOURCE2>/file_concat_1</SOURCE2>
</SOURCES>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>62</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>70</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -753,15 +753,15 @@
<INODEID>16392</INODEID>
<PATH>/file_symlink</PATH>
<VALUE>/file_concat_target</VALUE>
- <MTIME>1410224798537</MTIME>
- <ATIME>1410224798537</ATIME>
+ <MTIME>1412114468398</MTIME>
+ <ATIME>1412114468398</ATIME>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>511</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>63</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>71</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -772,19 +772,19 @@
<INODEID>16393</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224798540</MTIME>
- <ATIME>1410224798540</ATIME>
+ <MTIME>1412114468401</MTIME>
+ <ATIME>1412114468401</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_1374813776_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1474796918_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>64</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>72</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -840,7 +840,7 @@
<OPCODE>OP_REASSIGN_LEASE</OPCODE>
<DATA>
<TXID>65</TXID>
- <LEASEHOLDER>DFSClient_NONMAPREDUCE_1374813776_1</LEASEHOLDER>
+ <LEASEHOLDER>DFSClient_NONMAPREDUCE_1474796918_1</LEASEHOLDER>
<PATH>/hard-lease-recovery-test</PATH>
<NEWHOLDER>HDFS_NameNode</NEWHOLDER>
</DATA>
@@ -853,8 +853,8 @@
<INODEID>0</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1410224801265</MTIME>
- <ATIME>1410224798540</ATIME>
+ <MTIME>1412114470807</MTIME>
+ <ATIME>1412114468401</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -865,7 +865,7 @@
<GENSTAMP>1011</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>aagarwal</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -876,13 +876,13 @@
<DATA>
<TXID>67</TXID>
<POOLNAME>pool1</POOLNAME>
- <OWNERNAME>jing</OWNERNAME>
+ <OWNERNAME>aagarwal</OWNERNAME>
<GROUPNAME>staff</GROUPNAME>
<MODE>493</MODE>
<LIMIT>9223372036854775807</LIMIT>
<MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>71</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>79</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -891,8 +891,8 @@
<TXID>68</TXID>
<POOLNAME>pool1</POOLNAME>
<LIMIT>99</LIMIT>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>72</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>80</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -903,9 +903,9 @@
<PATH>/path</PATH>
<REPLICATION>1</REPLICATION>
<POOL>pool1</POOL>
- <EXPIRATION>2305844419438495525</EXPIRATION>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>73</RPC_CALLID>
+ <EXPIRATION>2305844421328165416</EXPIRATION>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>81</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -914,8 +914,8 @@
<TXID>70</TXID>
<ID>1</ID>
<REPLICATION>2</REPLICATION>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>74</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>82</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -923,8 +923,8 @@
<DATA>
<TXID>71</TXID>
<ID>1</ID>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>75</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>83</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -932,8 +932,8 @@
<DATA>
<TXID>72</TXID>
<POOLNAME>pool1</POOLNAME>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>76</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>84</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -979,8 +979,8 @@
<NAME>a1</NAME>
<VALUE>0x313233</VALUE>
</XATTR>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>78</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>86</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -993,8 +993,8 @@
<NAME>a2</NAME>
<VALUE>0x373839</VALUE>
</XATTR>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>79</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>87</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -1006,22 +1006,22 @@
<NAMESPACE>USER</NAMESPACE>
<NAME>a2</NAME>
</XATTR>
- <RPC_CLIENTID>b53e8d0a-8d92-4067-b8c8-637ac951bac2</RPC_CLIENTID>
- <RPC_CALLID>80</RPC_CALLID>
+ <RPC_CLIENTID>0a28b871-f75a-46a4-80e0-fe41cbb6b034</RPC_CLIENTID>
+ <RPC_CALLID>88</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
<DATA>
<TXID>77</TXID>
- <STARTTIME>1410224801616</STARTTIME>
+ <STARTTIME>1412114471510</STARTTIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
<DATA>
<TXID>78</TXID>
- <FINALIZETIME>1410224801616</FINALIZETIME>
+ <FINALIZETIME>1412114471510</FINALIZETIME>
</DATA>
</RECORD>
<RECORD>
[3/3] git commit: HDFS-7171. Fix Jenkins failures in HDFS-6581
branch. (Arpit Agarwal)
Posted by ar...@apache.org.
HDFS-7171. Fix Jenkins failures in HDFS-6581 branch. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a45ad330
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a45ad330
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a45ad330
Branch: refs/heads/HDFS-6581
Commit: a45ad330facc56f06ed42eb71304c49ef56dc549
Parents: 65ef35b
Author: arp <ar...@apache.org>
Authored: Tue Sep 30 18:25:20 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Tue Sep 30 18:25:20 2014 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-6581.txt | 1 +
.../hdfs/server/namenode/FSNamesystem.java | 1 -
.../hadoop/hdfs/TestBlockStoragePolicy.java | 7 ++-
.../hdfs/server/mover/TestStorageMover.java | 52 --------------------
4 files changed, 6 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a45ad330/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
index 3be544a..b41e133 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
@@ -92,4 +92,5 @@
HDFS-7129. Metrics to track usage of memory for writes. (Xiaoyu Yao
via Arpit Agarwal)
+ HDFS-7171. Fix Jenkins failures in HDFS-6581 branch. (Arpit Agarwal)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a45ad330/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 082e3bf..baea1a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2340,7 +2340,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
}
src = FSDirectory.resolvePath(src, pathComponents, dir);
- INode inode = dir.getINode(src);
// get the corresponding policy and make sure the policy name is valid
BlockStoragePolicy policy = blockManager.getStoragePolicy(policyName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a45ad330/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 771b7bd..03317b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -69,7 +69,7 @@ public class TestBlockStoragePolicy {
static final byte COLD = (byte) 4;
static final byte WARM = (byte) 8;
static final byte HOT = (byte) 12;
-
+ static final byte LAZY_PERSIST = (byte) 15;
@Test (timeout=300000)
public void testConfigKeyEnabled() throws IOException {
@@ -116,6 +116,9 @@ public class TestBlockStoragePolicy {
expectedPolicyStrings.put(HOT,
"BlockStoragePolicy{HOT:12, storageTypes=[DISK], " +
"creationFallbacks=[], replicationFallbacks=[ARCHIVE]}");
+ expectedPolicyStrings.put(LAZY_PERSIST,
+ "BlockStoragePolicy{LAZY_PERSIST:15, storageTypes=[RAM_DISK, DISK], " +
+ "creationFallbacks=[DISK], replicationFallbacks=[DISK]}");
for(byte i = 1; i < 16; i++) {
final BlockStoragePolicy policy = POLICY_SUITE.getPolicy(i);
@@ -1141,7 +1144,7 @@ public class TestBlockStoragePolicy {
final DistributedFileSystem fs = cluster.getFileSystem();
try {
BlockStoragePolicy[] policies = fs.getStoragePolicies();
- Assert.assertEquals(3, policies.length);
+ Assert.assertEquals(4, policies.length);
Assert.assertEquals(POLICY_SUITE.getPolicy(COLD).toString(),
policies[0].toString());
Assert.assertEquals(POLICY_SUITE.getPolicy(WARM).toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a45ad330/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index 0f8a9f6..2dae239 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -793,56 +793,4 @@ public class TestStorageMover {
test.shutdownCluster();
}
}
-
- /**
- * Test blocks of lazy_persist file on RAM_DISK will not be moved to other
- * storage types by the Storage Mover.
- */
- @Test
- public void testRamDiskNotMoved() throws Exception {
- LOG.info("testRamDiskNotMoved");
- final PathPolicyMap pathPolicyMap = new PathPolicyMap(0);
- final NamespaceScheme nsScheme = pathPolicyMap.newNamespaceScheme();
-
- final long diskCapacity = 100 * BLOCK_SIZE;
- final long archiveCapacity = (6 + HdfsConstants.MIN_BLOCKS_FOR_WRITE)
- * BLOCK_SIZE;
- final long ramDiskCapacity = 10 * BLOCK_SIZE;
- final long[][] capacities = genCapacities(1, 0, 0, 1,
- diskCapacity, archiveCapacity, ramDiskCapacity);
- final int LAZY_WRITER_INTERVAL_SEC = 1;
- final ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
- 1, (short)1, genStorageTypes(1, 0, 0, 1), capacities);
- clusterScheme.conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC,
- LAZY_WRITER_INTERVAL_SEC);
- final MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
-
- try {
- test.runBasicTest(false);
-
- // test creating a hot RAM_DISK file
- final int SEED = 0xFADED;
- final Path foo_hot = new Path(pathPolicyMap.hot, "foo_hot");
- DFSTestUtil.createFile(test.dfs, foo_hot, true, BLOCK_SIZE, BLOCK_SIZE,
- BLOCK_SIZE, (short) 1, SEED, true);
- Assert.assertTrue(DFSTestUtil.verifyFileReplicasOnStorageType(test.dfs,
- test.dfs.getClient(), foo_hot, StorageType.RAM_DISK));
-
- // Sleep for a short time to allow the lazy writer thread to do its job
- Thread.sleep(6 * LAZY_WRITER_INTERVAL_SEC * 1000);
-
- // Verify policy related name change is allowed
- final Path foo_hot_new = new Path(pathPolicyMap.warm, "foo_hot");
- test.dfs.rename(foo_hot, pathPolicyMap.warm);
- Assert.assertTrue(test.dfs.exists(foo_hot_new));
-
- // Verify blocks on ram disk will not be moved to other storage types by
- // policy based Storage Mover.
- test.migrate();
- Assert.assertTrue(DFSTestUtil.verifyFileReplicasOnStorageType(test.dfs,
- test.dfs.getClient(), foo_hot_new, StorageType.RAM_DISK));
- } finally {
- test.shutdownCluster();
- }
- }
}
[2/3] git commit: Merge branch 'trunk' into HDFS-6581
Posted by ar...@apache.org.
Merge branch 'trunk' into HDFS-6581
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65ef35bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65ef35bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65ef35bb
Branch: refs/heads/HDFS-6581
Commit: 65ef35bbabefc665897589bf92feae2ebf9da10f
Parents: dde2ed1 9e9e9cf
Author: arp <ar...@apache.org>
Authored: Tue Sep 30 18:20:13 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Tue Sep 30 18:20:13 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSDirectory.java | 5 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 4999 -> 5065 bytes
.../src/test/resources/editsStored.xml | 240 ++++++++-----------
4 files changed, 111 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/65ef35bb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------