You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/10/01 18:06:30 UTC
[39/50] [abbrv] git commit: HDFS-7153. Add storagePolicy to NN edit
log during file creation. (Arpit Agarwal)
HDFS-7153. Add storagePolicy to NN edit log during file creation. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d45e7c7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d45e7c7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d45e7c7e
Branch: refs/heads/trunk
Commit: d45e7c7e856c7103752888c0395fa94985cd7670
Parents: 9a53c36
Author: arp <ar...@apache.org>
Authored: Mon Sep 29 12:36:27 2014 -0700
Committer: arp <ar...@apache.org>
Committed: Mon Sep 29 12:36:27 2014 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-6581.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSDirectory.java | 16 ++++++++++++----
.../hadoop/hdfs/server/namenode/FSEditLog.java | 3 ++-
.../hdfs/server/namenode/FSEditLogLoader.java | 2 +-
.../hadoop/hdfs/server/namenode/FSEditLogOp.java | 16 ++++++++++++++++
.../hdfs/server/namenode/FSImageSerialization.java | 10 ++++++++++
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 2 +-
7 files changed, 45 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45e7c7e/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
index b7c2b22..d83af6c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
@@ -83,3 +83,6 @@
HDFS-7155. Bugfix in createLocatedFileStatus caused by bad merge.
(Arpit Agarwal)
+ HDFS-7153. Add storagePolicy to NN edit log during file creation.
+ (Arpit Agarwal)
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45e7c7e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 80d37ca..d1ac5ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -279,9 +279,16 @@ public class FSDirectory implements Closeable {
private static INodeFile newINodeFile(long id, PermissionStatus permissions,
long mtime, long atime, short replication, long preferredBlockSize,
boolean isLazyPersist) {
+ return newINodeFile(id, permissions, mtime, atime, replication, preferredBlockSize,
+ isLazyPersist, (byte)0);
+ }
+
+ private static INodeFile newINodeFile(long id, PermissionStatus permissions,
+ long mtime, long atime, short replication, long preferredBlockSize,
+ boolean isLazyPersist, byte storagePolicyId) {
return new INodeFile(id, null, permissions, mtime, atime,
BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize,
- isLazyPersist, (byte) 0);
+ isLazyPersist, storagePolicyId);
}
/**
@@ -334,17 +341,18 @@ public class FSDirectory implements Closeable {
boolean isLazyPersist,
boolean underConstruction,
String clientName,
- String clientMachine) {
+ String clientMachine,
+ byte storagePolicyId) {
final INodeFile newNode;
assert hasWriteLock();
if (underConstruction) {
newNode = newINodeFile(id, permissions, modificationTime,
- modificationTime, replication, preferredBlockSize, isLazyPersist);
+ modificationTime, replication, preferredBlockSize, isLazyPersist, storagePolicyId);
newNode.toUnderConstruction(clientName, clientMachine);
} else {
newNode = newINodeFile(id, permissions, modificationTime, atime,
- replication, preferredBlockSize, isLazyPersist);
+ replication, preferredBlockSize, isLazyPersist, storagePolicyId);
}
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45e7c7e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 2cf3aff..114e109 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -720,7 +720,8 @@ public class FSEditLog implements LogsPurgeable {
.setClientName(newNode.getFileUnderConstructionFeature().getClientName())
.setClientMachine(
newNode.getFileUnderConstructionFeature().getClientMachine())
- .setOverwrite(overwrite);
+ .setOverwrite(overwrite)
+ .setStoragePolicyId(newNode.getStoragePolicyID());
AclFeature f = newNode.getAclFeature();
if (f != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45e7c7e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index d7dc2c0..a8b7662 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -367,7 +367,7 @@ public class FSEditLogLoader {
replication, addCloseOp.mtime, addCloseOp.atime,
addCloseOp.blockSize, addCloseOp.isLazyPersist,
true, addCloseOp.clientName,
- addCloseOp.clientMachine);
+ addCloseOp.clientMachine, addCloseOp.storagePolicyId);
fsNamesys.leaseManager.addLease(addCloseOp.clientName, path);
// add the op into retry cache if necessary
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45e7c7e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 01c5ab5..107a091 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -103,6 +103,7 @@ import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEditLogProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrEditLogProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException;
import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
@@ -411,6 +412,7 @@ public abstract class FSEditLogOp {
String clientName;
String clientMachine;
boolean overwrite;
+ byte storagePolicyId;
private AddCloseOp(FSEditLogOpCodes opCode) {
super(opCode);
@@ -501,6 +503,11 @@ public abstract class FSEditLogOp {
return (T)this;
}
+ <T extends AddCloseOp> T setStoragePolicyId(byte storagePolicyId) {
+ this.storagePolicyId = storagePolicyId;
+ return (T)this;
+ }
+
@Override
public void writeFields(DataOutputStream out) throws IOException {
FSImageSerialization.writeLong(inodeId, out);
@@ -521,6 +528,7 @@ public abstract class FSEditLogOp {
FSImageSerialization.writeString(clientName,out);
FSImageSerialization.writeString(clientMachine,out);
FSImageSerialization.writeBoolean(overwrite, out);
+ FSImageSerialization.writeByte(storagePolicyId, out);
// write clientId and callId
writeRpcIds(rpcClientId, rpcCallId, out);
}
@@ -599,6 +607,12 @@ public abstract class FSEditLogOp {
} else {
this.overwrite = false;
}
+ if (NameNodeLayoutVersion.supports(
+ NameNodeLayoutVersion.Feature.BLOCK_STORAGE_POLICY, logVersion)) {
+ this.storagePolicyId = FSImageSerialization.readByte(in);
+ } else {
+ this.storagePolicyId = BlockStoragePolicySuite.ID_UNSPECIFIED;
+ }
// read clientId and callId
readRpcIds(in, logVersion);
} else {
@@ -661,6 +675,8 @@ public abstract class FSEditLogOp {
if (this.opCode == OP_ADD) {
appendRpcIdsToString(builder, rpcClientId, rpcCallId);
}
+ builder.append(", storagePolicyId=");
+ builder.append(storagePolicyId);
builder.append(", opCode=");
builder.append(opCode);
builder.append(", txid=");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45e7c7e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 1e82399..16636a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -385,6 +385,12 @@ public class FSImageSerialization {
uBoolean.write(out);
}
+ /** write the byte value */
+ static void writeByte(byte value, DataOutputStream out)
+ throws IOException {
+ out.write(value);
+ }
+
/** read the int value */
static int readInt(DataInput in) throws IOException {
IntWritable uInt = TL_DATA.get().U_INT;
@@ -424,6 +430,10 @@ public class FSImageSerialization {
return bytes;
}
+ public static byte readByte(DataInput in) throws IOException {
+ return in.readByte();
+ }
+
/**
* Reading the path from the image and converting it to byte[][] directly
* this saves us an array copy and conversions to and from String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45e7c7e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 54b7acd..8cf4d61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2611,7 +2611,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
clientMachine, create, overwrite, createParent, replication,
blockSize, isLazyPersist, suite, protocolVersion, edek, logRetryCache);
stat = dir.getFileInfo(src, false,
- FSDirectory.isReservedRawName(srcArg), false);
+ FSDirectory.isReservedRawName(srcArg), true);
} catch (StandbyException se) {
skipSync = true;
throw se;