You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/06/03 01:47:33 UTC

[01/50] [abbrv] hadoop git commit: MAPREDUCE-6336. Enable v2 FileOutputCommitter by default. (Siqi Li via gera)

Repository: hadoop
Updated Branches:
  refs/heads/YARN-2928 a9738ceb1 -> 2e1248088


MAPREDUCE-6336. Enable v2 FileOutputCommitter by default. (Siqi Li via gera)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/315bde8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/315bde8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/315bde8e

Branch: refs/heads/YARN-2928
Commit: 315bde8e25f755cfabbc2069da87f3570b6016ae
Parents: 41d3f67
Author: Gera Shegalov <ge...@apache.org>
Authored: Wed May 27 13:11:10 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 15:46:58 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt                              | 3 +++
 .../src/main/resources/mapred-default.xml                         | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/315bde8e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 221d98c..c0df835 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -37,6 +37,9 @@ Trunk (Unreleased)
     MAPREDUCE-4424. 'mapred job -list' command should show the job name
     as well. (Avinash Kujur via aajisaka)
 
+    MAPREDUCE-6336. Enable v2 FileOutputCommitter by default.
+    (Siqi Li via gera)
+
   NEW FEATURES
 
     MAPREDUCE-778. Rumen Anonymizer. (Amar Kamat and Chris Douglas via amarrk)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/315bde8e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index acb45a7..bc41443 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1330,7 +1330,7 @@
 
 <property>
   <name>mapreduce.fileoutputcommitter.algorithm.version</name>
-  <value>1</value>
+  <value>2</value>
   <description>The file output committer algorithm version
   valid algorithm version number: 1 or 2
   default to 1, which is the original algorithm


[22/50] [abbrv] hadoop git commit: Fixed more FilesSystemRMStateStore issues. Contributed by Vinod Kumar Vavilapalli.

Posted by zj...@apache.org.
Fixed more FilesSystemRMStateStore issues. Contributed by Vinod Kumar Vavilapalli.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14b30520
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14b30520
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14b30520

Branch: refs/heads/YARN-2928
Commit: 14b30520f9ca9d0f5555f594430f70d45e6d60b6
Parents: 3fcb33b
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu May 28 15:25:12 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700

----------------------------------------------------------------------
 .../hadoop/yarn/conf/YarnConfiguration.java     | 12 +++++++++++
 .../recovery/FileSystemRMStateStore.java        | 20 +++++++++----------
 .../recovery/TestFSRMStateStore.java            | 21 ++++++++++----------
 3 files changed, 32 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b30520/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6892e2d..1d42c60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -269,6 +269,18 @@ public class YarnConfiguration extends Configuration {
   /** ACL used in case none is found. Allows nothing. */
   public static final String DEFAULT_YARN_APP_ACL = " ";
 
+  /**
+   * Enable/disable intermediate-data encryption at YARN level. For now, this
+   * only is used by the FileSystemRMStateStore to setup right file-system
+   * security attributes.
+   */
+  @Private
+  public static final String YARN_INTERMEDIATE_DATA_ENCRYPTION = YARN_PREFIX
+      + "intermediate-data-encryption.enable";
+
+  @Private
+  public static final Boolean DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION = false;
+
   /** The address of the RM admin interface.*/
   public static final String RM_ADMIN_ADDRESS = 
     RM_PREFIX + "admin.address";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b30520/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 0d97d6f..b972791 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AM
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
+
 import com.google.common.annotations.VisibleForTesting;
 
 @Private
@@ -100,7 +101,8 @@ public class FileSystemRMStateStore extends RMStateStore {
   private Path dtSequenceNumberPath = null;
   private int fsNumRetries;
   private long fsRetryInterval;
-  private volatile boolean isHDFS;
+  private boolean intermediateEncryptionEnabled =
+      YarnConfiguration.DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION;
 
   @VisibleForTesting
   Path fsWorkingPath;
@@ -121,6 +123,9 @@ public class FileSystemRMStateStore extends RMStateStore {
     fsRetryInterval =
         conf.getLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
                 YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_INTERVAL_MS);
+    intermediateEncryptionEnabled =
+        conf.getBoolean(YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION,
+          YarnConfiguration.DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION);
   }
 
   @Override
@@ -145,17 +150,11 @@ public class FileSystemRMStateStore extends RMStateStore {
     }
 
     fs = fsWorkingPath.getFileSystem(fsConf);
-    isHDFS = fs.getScheme().toLowerCase().contains("hdfs");
     mkdirsWithRetries(rmDTSecretManagerRoot);
     mkdirsWithRetries(rmAppRoot);
     mkdirsWithRetries(amrmTokenSecretManagerRoot);
   }
 
-  @VisibleForTesting
-  void setIsHDFS(boolean isHDFS) {
-    this.isHDFS = isHDFS;
-  }
-
   @Override
   protected synchronized void closeInternal() throws Exception {
     closeWithRetries();
@@ -856,10 +855,11 @@ public class FileSystemRMStateStore extends RMStateStore {
 
   private void setUnreadableBySuperuserXattrib(Path p)
           throws IOException {
-    if (isHDFS &&
-            !fs.getXAttrs(p).containsKey(UNREADABLE_BY_SUPERUSER_XATTRIB)) {
+    if (fs.getScheme().toLowerCase().contains("hdfs")
+        && intermediateEncryptionEnabled
+        && !fs.getXAttrs(p).containsKey(UNREADABLE_BY_SUPERUSER_XATTRIB)) {
       fs.setXAttr(p, UNREADABLE_BY_SUPERUSER_XATTRIB, null,
-              EnumSet.of(XAttrSetFlag.CREATE));
+        EnumSet.of(XAttrSetFlag.CREATE));
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b30520/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index 2dba8b4..c842294 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -26,11 +26,6 @@ import java.util.LinkedList;
 import java.util.Queue;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.Assert;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -38,17 +33,22 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
 import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class TestFSRMStateStore extends RMStateStoreTestBase {
@@ -111,6 +111,10 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
       conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
       conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
               900L);
+      if (adminCheckEnable) {
+        conf.setBoolean(
+          YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true);
+      }
       this.store = new TestFileSystemRMStore(conf);
       Assert.assertEquals(store.getNumRetries(), 8);
       Assert.assertEquals(store.getRetryInterval(), 900L);
@@ -119,11 +123,6 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
       store.startInternal();
       Assert.assertTrue(store.fs != previousFs);
       Assert.assertTrue(store.fs.getConf() == store.fsConf);
-      if (adminCheckEnable) {
-        store.setIsHDFS(true);
-      } else {
-        store.setIsHDFS(false);
-      }
       return store;
     }
 


[29/50] [abbrv] hadoop git commit: YARN-3716. Node-label-expression should be included by ResourceRequestPBImpl.toString. (Xianyin Xin via wangda)

Posted by zj...@apache.org.
YARN-3716. Node-label-expression should be included by ResourceRequestPBImpl.toString. (Xianyin Xin via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/219d73d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/219d73d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/219d73d8

Branch: refs/heads/YARN-2928
Commit: 219d73d826685145bf759cb51178f432a1b626d5
Parents: 14b3052
Author: Wangda Tan <wa...@apache.org>
Authored: Thu May 28 16:34:18 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                                   | 3 +++
 .../hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java    | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/219d73d8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1dd4ee3..2fe1b02 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -549,6 +549,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3632. Ordering policy should be allowed to reorder an application when
     demand changes. (Craig Welch via jianhe)
 
+    YARN-3716. Node-label-expression should be included by 
+    ResourceRequestPBImpl.toString. (Xianyin Xin via wangda)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/219d73d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index 791e6a3..fc09cef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -184,7 +184,8 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
     return "{Priority: " + getPriority() + ", Capability: " + getCapability()
         + ", # Containers: " + getNumContainers()
         + ", Location: " + getResourceName()
-        + ", Relax Locality: " + getRelaxLocality() + "}";
+        + ", Relax Locality: " + getRelaxLocality()
+        + ", Node Label Expression: " + getNodeLabelExpression() + "}";
   }
 
   @Override


[32/50] [abbrv] hadoop git commit: HDFS-8443. Document dfs.namenode.service.handler.count in hdfs-site.xml. Contributed by J.Andreina.

Posted by zj...@apache.org.
HDFS-8443. Document dfs.namenode.service.handler.count in hdfs-site.xml. Contributed by J.Andreina.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdc6ac8a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdc6ac8a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdc6ac8a

Branch: refs/heads/YARN-2928
Commit: fdc6ac8a1da468f098a32f8e15e812b6ec67a4d6
Parents: 219d73d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri May 29 11:36:08 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt         |  3 +++
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 16 +++++++++++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdc6ac8a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1c9b25e..514076b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -585,6 +585,9 @@ Release 2.8.0 - UNRELEASED
 
     HDFS-8482. Rename BlockInfoContiguous to BlockInfo. (Zhe Zhang via wang)
 
+    HDFS-8443. Document dfs.namenode.service.handler.count in hdfs-site.xml.
+    (J.Andreina via aajisaka)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdc6ac8a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 7f0730b..7b579cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -602,7 +602,21 @@
 <property>
   <name>dfs.namenode.handler.count</name>
   <value>10</value>
-  <description>The number of server threads for the namenode.</description>
+  <description>The number of Namenode RPC server threads that listen to
+  requests from clients.
+  If dfs.namenode.servicerpc-address is not configured then
+  Namenode RPC server threads listen to requests from all nodes.
+  </description>
+</property>
+
+<property>
+  <name>dfs.namenode.service.handler.count</name>
+  <value>10</value>
+  <description>The number of Namenode RPC server threads that listen to
+  requests from DataNodes and from all other non-client nodes.
+  dfs.namenode.service.handler.count will be valid only if
+  dfs.namenode.servicerpc-address is configured.
+  </description>
 </property>
 
 <property>


[04/50] [abbrv] hadoop git commit: HDFS-8482. Rename BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.

Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 476ff36..9acd81f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
@@ -942,7 +942,7 @@ public class FSEditLogLoader {
    */
   private void addNewBlock(FSDirectory fsDir, AddBlockOp op, INodeFile file)
       throws IOException {
-    BlockInfoContiguous[] oldBlocks = file.getBlocks();
+    BlockInfo[] oldBlocks = file.getBlocks();
     Block pBlock = op.getPenultimateBlock();
     Block newBlock= op.getLastBlock();
     
@@ -968,7 +968,7 @@ public class FSEditLogLoader {
       Preconditions.checkState(oldBlocks == null || oldBlocks.length == 0);
     }
     // add the new block
-    BlockInfoContiguous newBI = new BlockInfoContiguousUnderConstruction(
+    BlockInfo newBI = new BlockInfoContiguousUnderConstruction(
           newBlock, file.getPreferredBlockReplication());
     fsNamesys.getBlockManager().addBlockCollection(newBI, file);
     file.addBlock(newBI);
@@ -982,7 +982,7 @@ public class FSEditLogLoader {
   private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
       INodesInPath iip, INodeFile file) throws IOException {
     // Update its block list
-    BlockInfoContiguous[] oldBlocks = file.getBlocks();
+    BlockInfo[] oldBlocks = file.getBlocks();
     Block[] newBlocks = op.getBlocks();
     String path = op.getPath();
     
@@ -991,7 +991,7 @@ public class FSEditLogLoader {
     
     // First, update blocks in common
     for (int i = 0; i < oldBlocks.length && i < newBlocks.length; i++) {
-      BlockInfoContiguous oldBlock = oldBlocks[i];
+      BlockInfo oldBlock = oldBlocks[i];
       Block newBlock = newBlocks[i];
       
       boolean isLastBlock = i == newBlocks.length - 1;
@@ -1043,7 +1043,7 @@ public class FSEditLogLoader {
       // We're adding blocks
       for (int i = oldBlocks.length; i < newBlocks.length; i++) {
         Block newBlock = newBlocks[i];
-        BlockInfoContiguous newBI;
+        BlockInfo newBI;
         if (!op.shouldCompleteLastBlock()) {
           // TODO: shouldn't this only be true for the last block?
           // what about an old-version fsync() where fsync isn't called
@@ -1055,7 +1055,7 @@ public class FSEditLogLoader {
           // is only executed when loading edits written by prior
           // versions of Hadoop. Current versions always log
           // OP_ADD operations as each block is allocated.
-          newBI = new BlockInfoContiguous(newBlock,
+          newBI = new BlockInfo(newBlock,
               file.getPreferredBlockReplication());
         }
         fsNamesys.getBlockManager().addBlockCollection(newBI, file);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index ec2babd..a46f2b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -52,7 +52,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.LayoutFlags;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -686,7 +686,7 @@ public class FSImageFormat {
 
     public void updateBlocksMap(INodeFile file) {
       // Add file->block mapping
-      final BlockInfoContiguous[] blocks = file.getBlocks();
+      final BlockInfo[] blocks = file.getBlocks();
       if (blocks != null) {
         final BlockManager bm = namesystem.getBlockManager();
         for (int i = 0; i < blocks.length; i++) {
@@ -753,9 +753,9 @@ public class FSImageFormat {
       // file
       
       // read blocks
-      BlockInfoContiguous[] blocks = new BlockInfoContiguous[numBlocks];
+      BlockInfo[] blocks = new BlockInfo[numBlocks];
       for (int j = 0; j < numBlocks; j++) {
-        blocks[j] = new BlockInfoContiguous(replication);
+        blocks[j] = new BlockInfo(replication);
         blocks[j].readFields(in);
       }
 
@@ -775,7 +775,7 @@ public class FSImageFormat {
             clientMachine = FSImageSerialization.readString(in);
             // convert the last block to BlockUC
             if (blocks.length > 0) {
-              BlockInfoContiguous lastBlk = blocks[blocks.length - 1];
+              BlockInfo lastBlk = blocks[blocks.length - 1];
               blocks[blocks.length - 1] = new BlockInfoContiguousUnderConstruction(
                   lastBlk, replication);
             }
@@ -958,9 +958,9 @@ public class FSImageFormat {
         FileUnderConstructionFeature uc = cons.getFileUnderConstructionFeature();
         oldnode.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
         if (oldnode.numBlocks() > 0) {
-          BlockInfoContiguous ucBlock = cons.getLastBlock();
+          BlockInfo ucBlock = cons.getLastBlock();
           // we do not replace the inode, just replace the last block of oldnode
-          BlockInfoContiguous info = namesystem.getBlockManager().addBlockCollection(
+          BlockInfo info = namesystem.getBlockManager().addBlockCollection(
               ucBlock, oldnode);
           oldnode.setBlock(oldnode.numBlocks() - 1, info);
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index d966c69..29713a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -43,7 +42,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
@@ -211,7 +210,7 @@ public final class FSImageFormatPBINode {
 
     public static void updateBlocksMap(INodeFile file, BlockManager bm) {
       // Add file->block mapping
-      final BlockInfoContiguous[] blocks = file.getBlocks();
+      final BlockInfo[] blocks = file.getBlocks();
       if (blocks != null) {
         for (int i = 0; i < blocks.length; i++) {
           file.setBlock(i, bm.addBlockCollection(blocks[i], file));
@@ -324,9 +323,9 @@ public final class FSImageFormatPBINode {
       short replication = (short) f.getReplication();
       LoaderContext state = parent.getLoaderContext();
 
-      BlockInfoContiguous[] blocks = new BlockInfoContiguous[bp.size()];
+      BlockInfo[] blocks = new BlockInfo[bp.size()];
       for (int i = 0, e = bp.size(); i < e; ++i) {
-        blocks[i] = new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
+        blocks[i] = new BlockInfo(PBHelper.convert(bp.get(i)), replication);
       }
       final PermissionStatus permissions = loadPermission(f.getPermission(),
           parent.getLoaderContext().getStringTable());
@@ -352,7 +351,7 @@ public final class FSImageFormatPBINode {
         INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
         file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
         if (blocks.length > 0) {
-          BlockInfoContiguous lastBlk = file.getLastBlock();
+          BlockInfo lastBlk = file.getLastBlock();
           // replace the last block of file
           file.setBlock(file.numBlocks() - 1, new BlockInfoContiguousUnderConstruction(
               lastBlk, replication));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 1888d87..d790b8f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
@@ -126,12 +126,12 @@ public class FSImageSerialization {
     long preferredBlockSize = in.readLong();
   
     int numBlocks = in.readInt();
-    BlockInfoContiguous[] blocks = new BlockInfoContiguous[numBlocks];
+    BlockInfo[] blocks = new BlockInfo[numBlocks];
     Block blk = new Block();
     int i = 0;
     for (; i < numBlocks-1; i++) {
       blk.readFields(in);
-      blocks[i] = new BlockInfoContiguous(blk, blockReplication);
+      blocks[i] = new BlockInfo(blk, blockReplication);
     }
     // last block is UNDER_CONSTRUCTION
     if(numBlocks > 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index bfd6eba..798f8d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -203,7 +203,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretMan
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager.SecretManagerState;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
@@ -2016,7 +2016,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     }
 
     // Check if the file is already being truncated with the same length
-    final BlockInfoContiguous last = file.getLastBlock();
+    final BlockInfo last = file.getLastBlock();
     if (last != null && last.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
       final Block truncateBlock
           = ((BlockInfoContiguousUnderConstruction)last).getTruncateBlock();
@@ -2090,7 +2090,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     leaseManager.addLease(
         file.getFileUnderConstructionFeature().getClientName(), file.getId());
     boolean shouldRecoverNow = (newBlock == null);
-    BlockInfoContiguous oldBlock = file.getLastBlock();
+    BlockInfo oldBlock = file.getLastBlock();
     boolean shouldCopyOnTruncate = shouldCopyOnTruncate(file, oldBlock);
     if(newBlock == null) {
       newBlock = (shouldCopyOnTruncate) ? createNewBlock() :
@@ -2141,7 +2141,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
    * Defines if a replica needs to be copied on truncate or
    * can be truncated in place.
    */
-  boolean shouldCopyOnTruncate(INodeFile file, BlockInfoContiguous blk) {
+  boolean shouldCopyOnTruncate(INodeFile file, BlockInfo blk) {
     if(!isUpgradeFinalized()) {
       return true;
     }
@@ -2505,7 +2505,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
       recoverLeaseInternal(RecoverLeaseOp.APPEND_FILE, iip, src, holder,
                            clientMachine, false);
       
-      final BlockInfoContiguous lastBlock = myFile.getLastBlock();
+      final BlockInfo lastBlock = myFile.getLastBlock();
       // Check that the block has at least minimum replication.
       if(lastBlock != null && lastBlock.isComplete() &&
           !getBlockManager().isSufficientlyReplicated(lastBlock)) {
@@ -2561,7 +2561,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
         }
       }
     } else {
-      BlockInfoContiguous lastBlock = file.getLastBlock();
+      BlockInfo lastBlock = file.getLastBlock();
       if (lastBlock != null) {
         ExtendedBlock blk = new ExtendedBlock(this.getBlockPoolId(), lastBlock);
         ret = new LocatedBlock(blk, new DatanodeInfo[0]);
@@ -2603,7 +2603,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   /** Compute quota change for converting a complete block to a UC block */
   private QuotaCounts computeQuotaDeltaForUCBlock(INodeFile file) {
     final QuotaCounts delta = new QuotaCounts.Builder().build();
-    final BlockInfoContiguous lastBlock = file.getLastBlock();
+    final BlockInfo lastBlock = file.getLastBlock();
     if (lastBlock != null) {
       final long diff = file.getPreferredBlockSize() - lastBlock.getNumBytes();
       final short repl = file.getPreferredBlockReplication();
@@ -2740,7 +2740,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
                 op.getExceptionMessage(src, holder, clientMachine,
                     "lease recovery is in progress. Try again later."));
         } else {
-          final BlockInfoContiguous lastBlock = file.getLastBlock();
+          final BlockInfo lastBlock = file.getLastBlock();
           if (lastBlock != null
               && lastBlock.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
             throw new RecoveryInProgressException(
@@ -3065,10 +3065,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
           .getBlocks());
     } else {
       // check the penultimate block of this file
-      BlockInfoContiguous b = v.getPenultimateBlock();
+      BlockInfo b = v.getPenultimateBlock();
       return b == null ||
           blockManager.checkBlocksProperlyReplicated(
-              src, new BlockInfoContiguous[] { b });
+              src, new BlockInfo[] { b });
     }
   }
 
@@ -3242,7 +3242,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
 
     for (Block b : blocks.getToDeleteList()) {
       if (trackBlockCounts) {
-        BlockInfoContiguous bi = getStoredBlock(b);
+        BlockInfo bi = getStoredBlock(b);
         if (bi.isComplete()) {
           numRemovedComplete++;
           if (bi.numNodes() >= blockManager.minReplication) {
@@ -3466,10 +3466,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
 
     final INodeFile pendingFile = iip.getLastINode().asFile();
     int nrBlocks = pendingFile.numBlocks();
-    BlockInfoContiguous[] blocks = pendingFile.getBlocks();
+    BlockInfo[] blocks = pendingFile.getBlocks();
 
     int nrCompleteBlocks;
-    BlockInfoContiguous curBlock = null;
+    BlockInfo curBlock = null;
     for(nrCompleteBlocks = 0; nrCompleteBlocks < nrBlocks; nrCompleteBlocks++) {
       curBlock = blocks[nrCompleteBlocks];
       if(!curBlock.isComplete())
@@ -3504,9 +3504,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
 
     // The last block is not COMPLETE, and
     // that the penultimate block if exists is either COMPLETE or COMMITTED
-    final BlockInfoContiguous lastBlock = pendingFile.getLastBlock();
+    final BlockInfo lastBlock = pendingFile.getLastBlock();
     BlockUCState lastBlockState = lastBlock.getBlockUCState();
-    BlockInfoContiguous penultimateBlock = pendingFile.getPenultimateBlock();
+    BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
 
     // If penultimate block doesn't exist then its minReplication is met
     boolean penultimateBlockMinReplication = penultimateBlock == null ? true :
@@ -3650,7 +3650,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   }
 
   @VisibleForTesting
-  BlockInfoContiguous getStoredBlock(Block block) {
+  BlockInfo getStoredBlock(Block block) {
     return blockManager.getStoredBlock(block);
   }
   
@@ -3710,7 +3710,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   
       checkNameNodeSafeMode(
           "Cannot commitBlockSynchronization while in safe mode");
-      final BlockInfoContiguous storedBlock = getStoredBlock(
+      final BlockInfo storedBlock = getStoredBlock(
           ExtendedBlock.getLocalBlock(oldBlock));
       if (storedBlock == null) {
         if (deleteblock) {
@@ -3871,7 +3871,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
    * @throws IOException on error
    */
   @VisibleForTesting
-  String closeFileCommitBlocks(INodeFile pendingFile, BlockInfoContiguous storedBlock)
+  String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock)
       throws IOException {
     final INodesInPath iip = INodesInPath.fromINode(pendingFile);
     final String src = iip.getPath();
@@ -4162,7 +4162,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
 
         while (it.hasNext()) {
           Block b = it.next();
-          BlockInfoContiguous blockInfo = blockManager.getStoredBlock(b);
+          BlockInfo blockInfo = blockManager.getStoredBlock(b);
           if (blockInfo.getBlockCollection().getStoragePolicyID()
               == lpPolicy.getId()) {
             filesToDelete.add(blockInfo.getBlockCollection());
@@ -5105,7 +5105,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     SafeModeInfo safeMode = this.safeMode;
     if (safeMode == null) // mostly true
       return;
-    BlockInfoContiguous storedBlock = getStoredBlock(b);
+    BlockInfo storedBlock = getStoredBlock(b);
     if (storedBlock.isComplete()) {
       safeMode.decrementSafeBlockCount((short)blockManager.countNodes(b).liveReplicas());
     }
@@ -5665,7 +5665,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
         + "access token for block " + block);
     
     // check stored block state
-    BlockInfoContiguous storedBlock = getStoredBlock(ExtendedBlock.getLocalBlock(block));
+    BlockInfo storedBlock = getStoredBlock(ExtendedBlock.getLocalBlock(block));
     if (storedBlock == null || 
         storedBlock.getBlockUCState() != BlockUCState.UNDER_CONSTRUCTION) {
         throw new IOException(block + 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
index 1ebdde6..d07ae1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
 
@@ -58,7 +58,7 @@ public class FileUnderConstructionFeature implements INode.Feature {
    */
   void updateLengthOfLastBlock(INodeFile f, long lastBlockLength)
       throws IOException {
-    BlockInfoContiguous lastBlock = f.getLastBlock();
+    BlockInfo lastBlock = f.getLastBlock();
     assert (lastBlock != null) : "The last block for path "
         + f.getFullPathName() + " is null when updating its length";
     assert (lastBlock instanceof BlockInfoContiguousUnderConstruction)
@@ -74,7 +74,7 @@ public class FileUnderConstructionFeature implements INode.Feature {
    */
   void cleanZeroSizeBlock(final INodeFile f,
       final BlocksMapUpdateInfo collectedBlocks) {
-    final BlockInfoContiguous[] blocks = f.getBlocks();
+    final BlockInfo[] blocks = f.getBlocks();
     if (blocks != null && blocks.length > 0
         && blocks[blocks.length - 1] instanceof BlockInfoContiguousUnderConstruction) {
       BlockInfoContiguousUnderConstruction lastUC =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index a6ff6fb..294323c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
@@ -121,17 +121,17 @@ public class INodeFile extends INodeWithAdditionalFields
 
   private long header = 0L;
 
-  private BlockInfoContiguous[] blocks;
+  private BlockInfo[] blocks;
 
   INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
-            long atime, BlockInfoContiguous[] blklist, short replication,
+            long atime, BlockInfo[] blklist, short replication,
             long preferredBlockSize) {
     this(id, name, permissions, mtime, atime, blklist, replication,
          preferredBlockSize, (byte) 0);
   }
 
   INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
-      long atime, BlockInfoContiguous[] blklist, short replication,
+      long atime, BlockInfo[] blklist, short replication,
       long preferredBlockSize, byte storagePolicyID) {
     super(id, name, permissions, mtime, atime);
     header = HeaderFormat.toLong(preferredBlockSize, replication, storagePolicyID);
@@ -226,13 +226,13 @@ public class INodeFile extends INodeWithAdditionalFields
   }
 
   @Override // BlockCollection
-  public void setBlock(int index, BlockInfoContiguous blk) {
+  public void setBlock(int index, BlockInfo blk) {
     this.blocks[index] = blk;
   }
 
   @Override // BlockCollection, the file should be under construction
   public BlockInfoContiguousUnderConstruction setLastBlock(
-      BlockInfoContiguous lastBlock, DatanodeStorageInfo[] locations)
+      BlockInfo lastBlock, DatanodeStorageInfo[] locations)
       throws IOException {
     Preconditions.checkState(isUnderConstruction(),
         "file is no longer under construction");
@@ -265,7 +265,7 @@ public class INodeFile extends INodeWithAdditionalFields
     BlockInfoContiguousUnderConstruction uc =
         (BlockInfoContiguousUnderConstruction)blocks[size_1];
     //copy to a new list
-    BlockInfoContiguous[] newlist = new BlockInfoContiguous[size_1];
+    BlockInfo[] newlist = new BlockInfo[size_1];
     System.arraycopy(blocks, 0, newlist, 0, size_1);
     setBlocks(newlist);
     return uc;
@@ -420,16 +420,16 @@ public class INodeFile extends INodeWithAdditionalFields
 
   /** @return the blocks of the file. */
   @Override
-  public BlockInfoContiguous[] getBlocks() {
+  public BlockInfo[] getBlocks() {
     return this.blocks;
   }
 
   /** @return blocks of the file corresponding to the snapshot. */
-  public BlockInfoContiguous[] getBlocks(int snapshot) {
+  public BlockInfo[] getBlocks(int snapshot) {
     if(snapshot == CURRENT_STATE_ID || getDiffs() == null)
       return getBlocks();
     FileDiff diff = getDiffs().getDiffById(snapshot);
-    BlockInfoContiguous[] snapshotBlocks =
+    BlockInfo[] snapshotBlocks =
         diff == null ? getBlocks() : diff.getBlocks();
     if(snapshotBlocks != null)
       return snapshotBlocks;
@@ -441,7 +441,7 @@ public class INodeFile extends INodeWithAdditionalFields
 
   void updateBlockCollection() {
     if (blocks != null) {
-      for(BlockInfoContiguous b : blocks) {
+      for(BlockInfo b : blocks) {
         b.setBlockCollection(this);
       }
     }
@@ -457,8 +457,8 @@ public class INodeFile extends INodeWithAdditionalFields
       totalAddedBlocks += f.blocks.length;
     }
     
-    BlockInfoContiguous[] newlist =
-        new BlockInfoContiguous[size + totalAddedBlocks];
+    BlockInfo[] newlist =
+        new BlockInfo[size + totalAddedBlocks];
     System.arraycopy(this.blocks, 0, newlist, 0, size);
     
     for(INodeFile in: inodes) {
@@ -473,12 +473,12 @@ public class INodeFile extends INodeWithAdditionalFields
   /**
    * add a block to the block list
    */
-  void addBlock(BlockInfoContiguous newblock) {
+  void addBlock(BlockInfo newblock) {
     if (this.blocks == null) {
-      this.setBlocks(new BlockInfoContiguous[]{newblock});
+      this.setBlocks(new BlockInfo[]{newblock});
     } else {
       int size = this.blocks.length;
-      BlockInfoContiguous[] newlist = new BlockInfoContiguous[size + 1];
+      BlockInfo[] newlist = new BlockInfo[size + 1];
       System.arraycopy(this.blocks, 0, newlist, 0, size);
       newlist[size] = newblock;
       this.setBlocks(newlist);
@@ -486,7 +486,7 @@ public class INodeFile extends INodeWithAdditionalFields
   }
 
   /** Set the blocks. */
-  public void setBlocks(BlockInfoContiguous[] blocks) {
+  public void setBlocks(BlockInfo[] blocks) {
     this.blocks = blocks;
   }
 
@@ -538,7 +538,7 @@ public class INodeFile extends INodeWithAdditionalFields
 
   public void clearFile(ReclaimContext reclaimContext) {
     if (blocks != null && reclaimContext.collectedBlocks != null) {
-      for (BlockInfoContiguous blk : blocks) {
+      for (BlockInfo blk : blocks) {
         reclaimContext.collectedBlocks.addDeleteBlock(blk);
         blk.setBlockCollection(null);
       }
@@ -710,16 +710,16 @@ public class INodeFile extends INodeWithAdditionalFields
    */
   public final QuotaCounts storagespaceConsumed(BlockStoragePolicy bsp) {
     QuotaCounts counts = new QuotaCounts.Builder().build();
-    final Iterable<BlockInfoContiguous> blocks;
+    final Iterable<BlockInfo> blocks;
     FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
     if (sf == null) {
       blocks = Arrays.asList(getBlocks());
     } else {
       // Collect all distinct blocks
-      Set<BlockInfoContiguous> allBlocks = new HashSet<>(Arrays.asList(getBlocks()));
+      Set<BlockInfo> allBlocks = new HashSet<>(Arrays.asList(getBlocks()));
       List<FileDiff> diffs = sf.getDiffs().asList();
       for(FileDiff diff : diffs) {
-        BlockInfoContiguous[] diffBlocks = diff.getBlocks();
+        BlockInfo[] diffBlocks = diff.getBlocks();
         if (diffBlocks != null) {
           allBlocks.addAll(Arrays.asList(diffBlocks));
         }
@@ -728,7 +728,7 @@ public class INodeFile extends INodeWithAdditionalFields
     }
 
     final short replication = getPreferredBlockReplication();
-    for (BlockInfoContiguous b : blocks) {
+    for (BlockInfo b : blocks) {
       long blockSize = b.isComplete() ? b.getNumBytes() :
           getPreferredBlockSize();
       counts.addStorageSpace(blockSize * replication);
@@ -747,7 +747,7 @@ public class INodeFile extends INodeWithAdditionalFields
   /**
    * Return the penultimate allocated block for this file.
    */
-  BlockInfoContiguous getPenultimateBlock() {
+  BlockInfo getPenultimateBlock() {
     if (blocks == null || blocks.length <= 1) {
       return null;
     }
@@ -755,7 +755,7 @@ public class INodeFile extends INodeWithAdditionalFields
   }
 
   @Override
-  public BlockInfoContiguous getLastBlock() {
+  public BlockInfo getLastBlock() {
     return blocks == null || blocks.length == 0? null: blocks[blocks.length-1];
   }
 
@@ -782,7 +782,7 @@ public class INodeFile extends INodeWithAdditionalFields
    */
   public long collectBlocksBeyondMax(final long max,
       final BlocksMapUpdateInfo collectedBlocks) {
-    final BlockInfoContiguous[] oldBlocks = getBlocks();
+    final BlockInfo[] oldBlocks = getBlocks();
     if (oldBlocks == null)
       return 0;
     // find the minimum n such that the size of the first n blocks > max
@@ -814,17 +814,17 @@ public class INodeFile extends INodeWithAdditionalFields
   void computeQuotaDeltaForTruncate(
       long newLength, BlockStoragePolicy bsps,
       QuotaCounts delta) {
-    final BlockInfoContiguous[] blocks = getBlocks();
+    final BlockInfo[] blocks = getBlocks();
     if (blocks == null || blocks.length == 0) {
       return;
     }
 
     long size = 0;
-    for (BlockInfoContiguous b : blocks) {
+    for (BlockInfo b : blocks) {
       size += b.getNumBytes();
     }
 
-    BlockInfoContiguous[] sblocks = null;
+    BlockInfo[] sblocks = null;
     FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
     if (sf != null) {
       FileDiff diff = sf.getDiffs().getLast();
@@ -833,7 +833,7 @@ public class INodeFile extends INodeWithAdditionalFields
 
     for (int i = blocks.length - 1; i >= 0 && size > newLength;
          size -= blocks[i].getNumBytes(), --i) {
-      BlockInfoContiguous bi = blocks[i];
+      BlockInfo bi = blocks[i];
       long truncatedBytes;
       if (size - newLength < bi.getNumBytes()) {
         // Record a full block as the last block will be copied during
@@ -863,20 +863,20 @@ public class INodeFile extends INodeWithAdditionalFields
   }
 
   void truncateBlocksTo(int n) {
-    final BlockInfoContiguous[] newBlocks;
+    final BlockInfo[] newBlocks;
     if (n == 0) {
-      newBlocks = BlockInfoContiguous.EMPTY_ARRAY;
+      newBlocks = BlockInfo.EMPTY_ARRAY;
     } else {
-      newBlocks = new BlockInfoContiguous[n];
+      newBlocks = new BlockInfo[n];
       System.arraycopy(getBlocks(), 0, newBlocks, 0, n);
     }
     // set new blocks
     setBlocks(newBlocks);
   }
 
-  public void collectBlocksBeyondSnapshot(BlockInfoContiguous[] snapshotBlocks,
+  public void collectBlocksBeyondSnapshot(BlockInfo[] snapshotBlocks,
                                           BlocksMapUpdateInfo collectedBlocks) {
-    BlockInfoContiguous[] oldBlocks = getBlocks();
+    BlockInfo[] oldBlocks = getBlocks();
     if(snapshotBlocks == null || oldBlocks == null)
       return;
     // Skip blocks in common between the file and the snapshot
@@ -900,7 +900,7 @@ public class INodeFile extends INodeWithAdditionalFields
     FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
     if(sf == null)
       return;
-    BlockInfoContiguous[] snapshotBlocks =
+    BlockInfo[] snapshotBlocks =
         getDiffs().findEarlierSnapshotBlocks(snapshotId);
     if(snapshotBlocks == null)
       return;
@@ -914,12 +914,12 @@ public class INodeFile extends INodeWithAdditionalFields
   /**
    * @return true if the block is contained in a snapshot or false otherwise.
    */
-  boolean isBlockInLatestSnapshot(BlockInfoContiguous block) {
+  boolean isBlockInLatestSnapshot(BlockInfo block) {
     FileWithSnapshotFeature sf = this.getFileWithSnapshotFeature();
     if (sf == null || sf.getDiffs() == null) {
       return false;
     }
-    BlockInfoContiguous[] snapshotBlocks = getDiffs()
+    BlockInfo[] snapshotBlocks = getDiffs()
         .findEarlierSnapshotBlocks(getDiffs().getLastSnapshotId());
     return snapshotBlocks != null &&
         Arrays.asList(snapshotBlocks).contains(block);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 0806f82..f954a58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -33,7 +33,7 @@ import java.util.TreeMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.util.Daemon;
 
@@ -108,11 +108,11 @@ public class LeaseManager {
     for (Long id : getINodeIdWithLeases()) {
       final INodeFile cons = fsnamesystem.getFSDirectory().getInode(id).asFile();
       Preconditions.checkState(cons.isUnderConstruction());
-      BlockInfoContiguous[] blocks = cons.getBlocks();
+      BlockInfo[] blocks = cons.getBlocks();
       if(blocks == null) {
         continue;
       }
-      for(BlockInfoContiguous b : blocks) {
+      for(BlockInfo b : blocks) {
         if(!b.isComplete())
           numUCBlocks++;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 0daf367..39180b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -64,7 +64,7 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactor
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
@@ -243,7 +243,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
       //get blockInfo
       Block block = new Block(Block.getBlockId(blockId));
       //find which file this block belongs to
-      BlockInfoContiguous blockInfo = bm.getStoredBlock(block);
+      BlockInfo blockInfo = bm.getStoredBlock(block);
       if(blockInfo == null) {
         out.println("Block "+ blockId +" " + NONEXISTENT_STATUS);
         LOG.warn("Block "+ blockId + " " + NONEXISTENT_STATUS);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index c4cbbc1..363c208 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.namenode.AclEntryStatusFormat;
 import org.apache.hadoop.hdfs.server.namenode.AclFeature;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -239,13 +239,13 @@ public class FSImageFormatPBSnapshot {
         FileDiff diff = new FileDiff(pbf.getSnapshotId(), copy, null,
             pbf.getFileSize());
         List<BlockProto> bpl = pbf.getBlocksList();
-        BlockInfoContiguous[] blocks = new BlockInfoContiguous[bpl.size()];
+        BlockInfo[] blocks = new BlockInfo[bpl.size()];
         for(int j = 0, e = bpl.size(); j < e; ++j) {
           Block blk = PBHelper.convert(bpl.get(j));
-          BlockInfoContiguous storedBlock =  fsn.getBlockManager().getStoredBlock(blk);
+          BlockInfo storedBlock =  fsn.getBlockManager().getStoredBlock(blk);
           if(storedBlock == null) {
             storedBlock = fsn.getBlockManager().addBlockCollection(
-                new BlockInfoContiguous(blk, copy.getFileReplication()), file);
+                new BlockInfo(blk, copy.getFileReplication()), file);
           }
           blocks[j] = storedBlock;
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
index 48c5f33..52bf58e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
@@ -21,7 +21,7 @@ import java.io.DataOutput;
 import java.io.IOException;
 import java.util.Arrays;
 
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@@ -38,7 +38,7 @@ public class FileDiff extends
   /** The file size at snapshot creation time. */
   private final long fileSize;
   /** A copy of the INodeFile block list. Used in truncate. */
-  private BlockInfoContiguous[] blocks;
+  private BlockInfo[] blocks;
 
   FileDiff(int snapshotId, INodeFile file) {
     super(snapshotId, null, null);
@@ -64,7 +64,7 @@ public class FileDiff extends
    * up to the current {@link #fileSize}.
    * Should be done only once.
    */
-  public void setBlocks(BlockInfoContiguous[] blocks) {
+  public void setBlocks(BlockInfo[] blocks) {
     if(this.blocks != null)
       return;
     int numBlocks = 0;
@@ -73,7 +73,7 @@ public class FileDiff extends
     this.blocks = Arrays.copyOf(blocks, numBlocks);
   }
 
-  public BlockInfoContiguous[] getBlocks() {
+  public BlockInfo[] getBlocks() {
     return blocks;
   }
 
@@ -118,7 +118,7 @@ public class FileDiff extends
     if (blocks == null || collectedBlocks == null) {
       return;
     }
-    for (BlockInfoContiguous blk : blocks) {
+    for (BlockInfo blk : blocks) {
       collectedBlocks.addDeleteBlock(blk);
     }
     blocks = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
index 62aaccd..6b8388e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
@@ -21,7 +21,7 @@ import java.util.Collections;
 import java.util.List;
 
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.namenode.INode;
@@ -59,14 +59,14 @@ public class FileDiffList extends
     }
   }
 
-  public BlockInfoContiguous[] findEarlierSnapshotBlocks(int snapshotId) {
+  public BlockInfo[] findEarlierSnapshotBlocks(int snapshotId) {
     assert snapshotId != Snapshot.NO_SNAPSHOT_ID : "Wrong snapshot id";
     if (snapshotId == Snapshot.CURRENT_STATE_ID) {
       return null;
     }
     List<FileDiff> diffs = this.asList();
     int i = Collections.binarySearch(diffs, snapshotId);
-    BlockInfoContiguous[] blocks = null;
+    BlockInfo[] blocks = null;
     for(i = i >= 0 ? i : -i-2; i >= 0; i--) {
       blocks = diffs.get(i).getBlocks();
       if(blocks != null) {
@@ -76,14 +76,14 @@ public class FileDiffList extends
     return blocks;
   }
 
-  public BlockInfoContiguous[] findLaterSnapshotBlocks(int snapshotId) {
+  public BlockInfo[] findLaterSnapshotBlocks(int snapshotId) {
     assert snapshotId != Snapshot.NO_SNAPSHOT_ID : "Wrong snapshot id";
     if (snapshotId == Snapshot.CURRENT_STATE_ID) {
       return null;
     }
     List<FileDiff> diffs = this.asList();
     int i = Collections.binarySearch(diffs, snapshotId);
-    BlockInfoContiguous[] blocks = null;
+    BlockInfo[] blocks = null;
     for (i = i >= 0 ? i+1 : -i-1; i < diffs.size(); i++) {
       blocks = diffs.get(i).getBlocks();
       if (blocks != null) {
@@ -100,7 +100,7 @@ public class FileDiffList extends
    */
   void combineAndCollectSnapshotBlocks(
       INode.ReclaimContext reclaimContext, INodeFile file, FileDiff removed) {
-    BlockInfoContiguous[] removedBlocks = removed.getBlocks();
+    BlockInfo[] removedBlocks = removed.getBlocks();
     if (removedBlocks == null) {
       FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature();
       assert sf != null : "FileWithSnapshotFeature is null";
@@ -114,10 +114,10 @@ public class FileDiffList extends
     if (earlierDiff != null) {
       earlierDiff.setBlocks(removedBlocks);
     }
-    BlockInfoContiguous[] earlierBlocks =
-        (earlierDiff == null ? new BlockInfoContiguous[]{} : earlierDiff.getBlocks());
+    BlockInfo[] earlierBlocks =
+        (earlierDiff == null ? new BlockInfo[]{} : earlierDiff.getBlocks());
     // Find later snapshot (or file itself) with blocks
-    BlockInfoContiguous[] laterBlocks = findLaterSnapshotBlocks(removed.getSnapshotId());
+    BlockInfo[] laterBlocks = findLaterSnapshotBlocks(removed.getSnapshotId());
     laterBlocks = (laterBlocks==null) ? file.getBlocks() : laterBlocks;
     // Skip blocks, which belong to either the earlier or the later lists
     int i = 0;
@@ -129,7 +129,7 @@ public class FileDiffList extends
       break;
     }
     // Check if last block is part of truncate recovery
-    BlockInfoContiguous lastBlock = file.getLastBlock();
+    BlockInfo lastBlock = file.getLastBlock();
     Block dontRemoveBlock = null;
     if (lastBlock != null && lastBlock.getBlockUCState().equals(
         HdfsServerConstants.BlockUCState.UNDER_RECOVERY)) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
index 555a662..f017d4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
@@ -22,7 +22,7 @@ import java.util.List;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.namenode.AclFeature;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.AclStorage;
@@ -208,7 +208,7 @@ public class FileWithSnapshotFeature implements INode.Feature {
 
     // Collect blocks that should be deleted
     FileDiff last = diffs.getLast();
-    BlockInfoContiguous[] snapshotBlocks = last == null ? null : last.getBlocks();
+    BlockInfo[] snapshotBlocks = last == null ? null : last.getBlocks();
     if(snapshotBlocks == null)
       file.collectBlocksBeyondMax(max, reclaimContext.collectedBlocks());
     else

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index a88a459..50b85c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -108,7 +108,7 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
@@ -1609,7 +1609,7 @@ public class DFSTestUtil {
   public static DatanodeDescriptor getExpectedPrimaryNode(NameNode nn,
       ExtendedBlock blk) {
     BlockManager bm0 = nn.getNamesystem().getBlockManager();
-    BlockInfoContiguous storedBlock = bm0.getStoredBlock(blk.getLocalBlock());
+    BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock());
     assertTrue("Block " + blk + " should be under construction, " +
         "got: " + storedBlock,
         storedBlock instanceof BlockInfoContiguousUnderConstruction);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index 1ab7427..1f18014 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
@@ -671,7 +671,7 @@ public class TestDecommission {
       GenericTestUtils.waitFor(new Supplier<Boolean>() {
         @Override
         public Boolean get() {
-          BlockInfoContiguous info =
+          BlockInfo info =
               blockManager.getStoredBlock(b.getLocalBlock());
           int count = 0;
           StringBuilder sb = new StringBuilder("Replica locations: ");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index c5662ca..96d4d8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -49,7 +49,7 @@ public class TestBlockInfo {
 
   @Test
   public void testIsDeleted() {
-    BlockInfoContiguous blockInfo = new BlockInfoContiguous((short) 3);
+    BlockInfo blockInfo = new BlockInfo((short) 3);
     BlockCollection bc = Mockito.mock(BlockCollection.class);
     blockInfo.setBlockCollection(bc);
     Assert.assertFalse(blockInfo.isDeleted());
@@ -59,7 +59,7 @@ public class TestBlockInfo {
 
   @Test
   public void testAddStorage() throws Exception {
-    BlockInfoContiguous blockInfo = new BlockInfoContiguous((short) 3);
+    BlockInfo blockInfo = new BlockInfo((short) 3);
 
     final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1");
 
@@ -71,9 +71,9 @@ public class TestBlockInfo {
 
   @Test
   public void testCopyConstructor() {
-    BlockInfoContiguous old = new BlockInfoContiguous((short) 3);
+    BlockInfo old = new BlockInfo((short) 3);
     try {
-      BlockInfoContiguous copy = new BlockInfoContiguous(old);
+      BlockInfo copy = new BlockInfo(old);
       assertEquals(old.getBlockCollection(), copy.getBlockCollection());
       assertEquals(old.getCapacity(), copy.getCapacity());
     } catch (Exception e) {
@@ -88,11 +88,11 @@ public class TestBlockInfo {
     final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1", "127.0.0.1");
     final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2"));
     final int NUM_BLOCKS = 10;
-    BlockInfoContiguous[] blockInfos = new BlockInfoContiguous[NUM_BLOCKS];
+    BlockInfo[] blockInfos = new BlockInfo[NUM_BLOCKS];
 
     // Create a few dummy blocks and add them to the first storage.
     for (int i = 0; i < NUM_BLOCKS; ++i) {
-      blockInfos[i] = new BlockInfoContiguous((short) 3);
+      blockInfos[i] = new BlockInfo((short) 3);
       storage1.addBlock(blockInfos[i]);
     }
 
@@ -111,14 +111,14 @@ public class TestBlockInfo {
 
     DatanodeStorageInfo dd = DFSTestUtil.createDatanodeStorageInfo("s1", "1.1.1.1");
     ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
-    ArrayList<BlockInfoContiguous> blockInfoList = new ArrayList<BlockInfoContiguous>();
+    ArrayList<BlockInfo> blockInfoList = new ArrayList<BlockInfo>();
     int headIndex;
     int curIndex;
 
     LOG.info("Building block list...");
     for (int i = 0; i < MAX_BLOCKS; i++) {
       blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
-      blockInfoList.add(new BlockInfoContiguous(blockList.get(i), (short) 3));
+      blockInfoList.add(new BlockInfo(blockList.get(i), (short) 3));
       dd.addBlock(blockInfoList.get(i));
 
       // index of the datanode should be 0
@@ -129,7 +129,7 @@ public class TestBlockInfo {
     // list length should be equal to the number of blocks we inserted
     LOG.info("Checking list length...");
     assertEquals("Length should be MAX_BLOCK", MAX_BLOCKS, dd.numBlocks());
-    Iterator<BlockInfoContiguous> it = dd.getBlockIterator();
+    Iterator<BlockInfo> it = dd.getBlockIterator();
     int len = 0;
     while (it.hasNext()) {
       it.next();
@@ -151,7 +151,7 @@ public class TestBlockInfo {
     // move head of the list to the head - this should not change the list
     LOG.info("Moving head to the head...");
 
-    BlockInfoContiguous temp = dd.getBlockListHeadForTesting();
+    BlockInfo temp = dd.getBlockListHeadForTesting();
     curIndex = 0;
     headIndex = 0;
     dd.moveBlockToHead(temp, curIndex, headIndex);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 58210c1..bf46ed7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -147,7 +147,7 @@ public class TestBlockManager {
   private void doBasicTest(int testIndex) {
     List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
     List<DatanodeDescriptor> origNodes = getNodes(origStorages);
-    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
 
     DatanodeStorageInfo[] pipeline = scheduleSingleReplication(blockInfo);
     assertEquals(2, pipeline.length);
@@ -179,7 +179,7 @@ public class TestBlockManager {
     // Block originally on A1, A2, B1
     List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
     List<DatanodeDescriptor> origNodes = getNodes(origStorages);
-    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
     
     // Decommission two of the nodes (A1, A2)
     List<DatanodeDescriptor> decomNodes = startDecommission(0, 1);
@@ -223,7 +223,7 @@ public class TestBlockManager {
     // Block originally on A1, A2, B1
     List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
     List<DatanodeDescriptor> origNodes = getNodes(origStorages);
-    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
     
     // Decommission all of the nodes
     List<DatanodeDescriptor> decomNodes = startDecommission(0, 1, 3);
@@ -276,7 +276,7 @@ public class TestBlockManager {
     // Block originally on A1, A2, B1
     List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
     List<DatanodeDescriptor> origNodes = getNodes(origStorages);
-    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
     
     // Decommission all of the nodes in rack A
     List<DatanodeDescriptor> decomNodes = startDecommission(0, 1, 2);
@@ -335,7 +335,7 @@ public class TestBlockManager {
   private void doTestSufficientlyReplBlocksUsesNewRack(int testIndex) {
     // Originally on only nodes in rack A.
     List<DatanodeDescriptor> origNodes = rackA;
-    BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+    BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
     DatanodeStorageInfo pipeline[] = scheduleSingleReplication(blockInfo);
     
     assertEquals(2, pipeline.length); // single new copy
@@ -378,7 +378,7 @@ public class TestBlockManager {
    * Tell the block manager that replication is completed for the given
    * pipeline.
    */
-  private void fulfillPipeline(BlockInfoContiguous blockInfo,
+  private void fulfillPipeline(BlockInfo blockInfo,
       DatanodeStorageInfo[] pipeline) throws IOException {
     for (int i = 1; i < pipeline.length; i++) {
       DatanodeStorageInfo storage = pipeline[i];
@@ -387,9 +387,9 @@ public class TestBlockManager {
     }
   }
 
-  private BlockInfoContiguous blockOnNodes(long blkId, List<DatanodeDescriptor> nodes) {
+  private BlockInfo blockOnNodes(long blkId, List<DatanodeDescriptor> nodes) {
     Block block = new Block(blkId);
-    BlockInfoContiguous blockInfo = new BlockInfoContiguous(block, (short) 3);
+    BlockInfo blockInfo = new BlockInfo(block, (short) 3);
 
     for (DatanodeDescriptor dn : nodes) {
       for (DatanodeStorageInfo storage : dn.getStorageInfos()) {
@@ -431,10 +431,10 @@ public class TestBlockManager {
     return nodes;
   }
   
-  private BlockInfoContiguous addBlockOnNodes(long blockId, List<DatanodeDescriptor> nodes) {
+  private BlockInfo addBlockOnNodes(long blockId, List<DatanodeDescriptor> nodes) {
     BlockCollection bc = Mockito.mock(BlockCollection.class);
     Mockito.doReturn((short)3).when(bc).getPreferredBlockReplication();
-    BlockInfoContiguous blockInfo = blockOnNodes(blockId, nodes);
+    BlockInfo blockInfo = blockOnNodes(blockId, nodes);
 
     bm.blocksMap.addBlockCollection(blockInfo, bc);
     return blockInfo;
@@ -677,21 +677,21 @@ public class TestBlockManager {
 
     // blk_42 is finalized.
     long receivedBlockId = 42;  // arbitrary
-    BlockInfoContiguous receivedBlock = addBlockToBM(receivedBlockId);
+    BlockInfo receivedBlock = addBlockToBM(receivedBlockId);
     rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivedBlock),
         ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
     builder.add(new FinalizedReplica(receivedBlock, null, null));
 
     // blk_43 is under construction.
     long receivingBlockId = 43;
-    BlockInfoContiguous receivingBlock = addUcBlockToBM(receivingBlockId);
+    BlockInfo receivingBlock = addUcBlockToBM(receivingBlockId);
     rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingBlock),
         ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
     builder.add(new ReplicaBeingWritten(receivingBlock, null, null, null));
 
     // blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record.
     long receivingReceivedBlockId = 44;
-    BlockInfoContiguous receivingReceivedBlock = addBlockToBM(receivingReceivedBlockId);
+    BlockInfo receivingReceivedBlock = addBlockToBM(receivingReceivedBlockId);
     rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
         ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
     rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
@@ -709,7 +709,7 @@ public class TestBlockManager {
 
     // blk_46 exists in DN for a long time, so it's in full BR, but not in IBR.
     long existedBlockId = 46;
-    BlockInfoContiguous existedBlock = addBlockToBM(existedBlockId);
+    BlockInfo existedBlock = addBlockToBM(existedBlockId);
     builder.add(new FinalizedReplica(existedBlock, null, null));
 
     // process IBR and full BR
@@ -735,17 +735,17 @@ public class TestBlockManager {
         (ds) >= 0);
   }
 
-  private BlockInfoContiguous addBlockToBM(long blkId) {
+  private BlockInfo addBlockToBM(long blkId) {
     Block block = new Block(blkId);
-    BlockInfoContiguous blockInfo =
-        new BlockInfoContiguous(block, (short) 3);
+    BlockInfo blockInfo =
+        new BlockInfo(block, (short) 3);
     BlockCollection bc = Mockito.mock(BlockCollection.class);
     Mockito.doReturn((short) 3).when(bc).getPreferredBlockReplication();
     bm.blocksMap.addBlockCollection(blockInfo, bc);
     return blockInfo;
   }
 
-  private BlockInfoContiguous addUcBlockToBM(long blkId) {
+  private BlockInfo addUcBlockToBM(long blkId) {
     Block block = new Block(blkId);
     BlockInfoContiguousUnderConstruction blockInfo =
         new BlockInfoContiguousUnderConstruction(block, (short) 3);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
index 7cdb423..fe639e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
@@ -58,8 +58,8 @@ public class TestDatanodeDescriptor {
   public void testBlocksCounter() throws Exception {
     DatanodeDescriptor dd = BlockManagerTestUtil.getLocalDatanodeDescriptor(true);
     assertEquals(0, dd.numBlocks());
-    BlockInfoContiguous blk = new BlockInfoContiguous(new Block(1L), (short) 1);
-    BlockInfoContiguous blk1 = new BlockInfoContiguous(new Block(2L), (short) 2);
+    BlockInfo blk = new BlockInfo(new Block(1L), (short) 1);
+    BlockInfo blk1 = new BlockInfo(new Block(2L), (short) 2);
     DatanodeStorageInfo[] storages = dd.getStorageInfos();
     assertTrue(storages.length > 0);
     // add first block

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
index 259404e..844c5d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
@@ -159,7 +159,7 @@ public class TestPendingReplication {
         DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, TIMEOUT);
     MiniDFSCluster cluster = null;
     Block block;
-    BlockInfoContiguous blockInfo;
+    BlockInfo blockInfo;
     try {
       cluster =
           new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_COUNT).build();
@@ -178,7 +178,7 @@ public class TestPendingReplication {
       //
 
       block = new Block(1, 1, 0);
-      blockInfo = new BlockInfoContiguous(block, (short) 3);
+      blockInfo = new BlockInfo(block, (short) 3);
 
       pendingReplications.increment(block,
           DatanodeStorageInfo.toDatanodeDescriptors(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 3226578..f317723 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -34,7 +34,6 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.ThreadLocalRandom;
 
@@ -1219,7 +1218,7 @@ public class TestReplicationPolicy {
     chosenBlocks = underReplicatedBlocks.chooseUnderReplicatedBlocks(1);
     assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
 
-    final BlockInfoContiguous info = new BlockInfoContiguous(block1, (short) 1);
+    final BlockInfo info = new BlockInfo(block1, (short) 1);
     final BlockCollection mbc = mock(BlockCollection.class);
     when(mbc.getLastBlock()).thenReturn(info);
     when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1);
@@ -1241,12 +1240,12 @@ public class TestReplicationPolicy {
     when(dn.isDecommissioned()).thenReturn(true);
     when(storage.getState()).thenReturn(DatanodeStorage.State.NORMAL);
     when(storage.getDatanodeDescriptor()).thenReturn(dn);
-    when(storage.removeBlock(any(BlockInfoContiguous.class))).thenReturn(true);
-    when(storage.addBlock(any(BlockInfoContiguous.class))).thenReturn
+    when(storage.removeBlock(any(BlockInfo.class))).thenReturn(true);
+    when(storage.addBlock(any(BlockInfo.class))).thenReturn
         (DatanodeStorageInfo.AddBlockResult.ADDED);
     ucBlock.addStorage(storage);
 
-    when(mbc.setLastBlock((BlockInfoContiguous) any(), (DatanodeStorageInfo[]) any()))
+    when(mbc.setLastBlock((BlockInfo) any(), (DatanodeStorageInfo[]) any()))
     .thenReturn(ucBlock);
 
     bm.convertLastBlockToUnderConstruction(mbc, 0L);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index 0349251..3f96c0c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@ -23,7 +23,7 @@ import java.io.IOException;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.common.Storage;
 
@@ -66,10 +66,10 @@ public class CreateEditsLog {
     INodeDirectory dirInode = new INodeDirectory(inodeId.nextValue(), null, p,
       0L);
     editLog.logMkDir(BASE_PATH, dirInode);
-    BlockInfoContiguous[] blocks = new BlockInfoContiguous[blocksPerFile];
+    BlockInfo[] blocks = new BlockInfo[blocksPerFile];
     for (int iB = 0; iB < blocksPerFile; ++iB) {
       blocks[iB] = 
-       new BlockInfoContiguous(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
+       new BlockInfo(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
                                replication);
     }
     
@@ -97,7 +97,7 @@ public class CreateEditsLog {
         editLog.logMkDir(currentDir, dirInode);
       }
       INodeFile fileUc = new INodeFile(inodeId.nextValue(), null,
-          p, 0L, 0L, BlockInfoContiguous.EMPTY_ARRAY, replication, blockSize);
+          p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
       fileUc.toUnderConstruction("", "");
       editLog.logOpenFile(filePath, fileUc, false, false);
       editLog.logCloseFile(filePath, inode);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
index a417c3d..301ee25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.junit.After;
 import org.junit.Before;
@@ -87,21 +87,21 @@ public class TestAddBlock {
     
     // check file1
     INodeFile file1Node = fsdir.getINode4Write(file1.toString()).asFile();
-    BlockInfoContiguous[] file1Blocks = file1Node.getBlocks();
+    BlockInfo[] file1Blocks = file1Node.getBlocks();
     assertEquals(1, file1Blocks.length);
     assertEquals(BLOCKSIZE - 1, file1Blocks[0].getNumBytes());
     assertEquals(BlockUCState.COMPLETE, file1Blocks[0].getBlockUCState());
     
     // check file2
     INodeFile file2Node = fsdir.getINode4Write(file2.toString()).asFile();
-    BlockInfoContiguous[] file2Blocks = file2Node.getBlocks();
+    BlockInfo[] file2Blocks = file2Node.getBlocks();
     assertEquals(1, file2Blocks.length);
     assertEquals(BLOCKSIZE, file2Blocks[0].getNumBytes());
     assertEquals(BlockUCState.COMPLETE, file2Blocks[0].getBlockUCState());
     
     // check file3
     INodeFile file3Node = fsdir.getINode4Write(file3.toString()).asFile();
-    BlockInfoContiguous[] file3Blocks = file3Node.getBlocks();
+    BlockInfo[] file3Blocks = file3Node.getBlocks();
     assertEquals(2, file3Blocks.length);
     assertEquals(BLOCKSIZE, file3Blocks[0].getNumBytes());
     assertEquals(BlockUCState.COMPLETE, file3Blocks[0].getBlockUCState());
@@ -110,7 +110,7 @@ public class TestAddBlock {
     
     // check file4
     INodeFile file4Node = fsdir.getINode4Write(file4.toString()).asFile();
-    BlockInfoContiguous[] file4Blocks = file4Node.getBlocks();
+    BlockInfo[] file4Blocks = file4Node.getBlocks();
     assertEquals(2, file4Blocks.length);
     assertEquals(BLOCKSIZE, file4Blocks[0].getNumBytes());
     assertEquals(BlockUCState.COMPLETE, file4Blocks[0].getBlockUCState());
@@ -141,7 +141,7 @@ public class TestAddBlock {
       FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
       
       INodeFile fileNode = fsdir.getINode4Write(file1.toString()).asFile();
-      BlockInfoContiguous[] fileBlocks = fileNode.getBlocks();
+      BlockInfo[] fileBlocks = fileNode.getBlocks();
       assertEquals(2, fileBlocks.length);
       assertEquals(BLOCKSIZE, fileBlocks[0].getNumBytes());
       assertEquals(BlockUCState.COMPLETE, fileBlocks[0].getBlockUCState());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
index 1fbe160..f372bec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.hdfs.TestFileCreation;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
@@ -91,12 +91,12 @@ public class TestBlockUnderConstruction {
         " isUnderConstruction = " + inode.isUnderConstruction() +
         " expected to be " + isFileOpen,
         inode.isUnderConstruction() == isFileOpen);
-    BlockInfoContiguous[] blocks = inode.getBlocks();
+    BlockInfo[] blocks = inode.getBlocks();
     assertTrue("File does not have blocks: " + inode.toString(),
         blocks != null && blocks.length > 0);
     
     int idx = 0;
-    BlockInfoContiguous curBlock;
+    BlockInfo curBlock;
     // all blocks but the last two should be regular blocks
     for(; idx < blocks.length - 2; idx++) {
       curBlock = blocks[idx];

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index ea560fe..8c9da01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -78,7 +78,7 @@ public class TestCommitBlockSynchronization {
     doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class));
     doReturn(blockInfo).when(file).getLastBlock();
     doReturn("").when(namesystemSpy).closeFileCommitBlocks(
-        any(INodeFile.class), any(BlockInfoContiguous.class));
+        any(INodeFile.class), any(BlockInfo.class));
     doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog();
 
     return namesystemSpy;
@@ -106,7 +106,7 @@ public class TestCommitBlockSynchronization {
         lastBlock, genStamp, length, false, false, newTargets, null);
 
     // Simulate 'completing' the block.
-    BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
+    BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
     completedBlockInfo.setBlockCollection(file);
     completedBlockInfo.setGenerationStamp(genStamp);
     doReturn(completedBlockInfo).when(namesystemSpy)
@@ -178,7 +178,7 @@ public class TestCommitBlockSynchronization {
     namesystemSpy.commitBlockSynchronization(
         lastBlock, genStamp, length, true, false, newTargets, null);
 
-    BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
+    BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
     completedBlockInfo.setBlockCollection(file);
     completedBlockInfo.setGenerationStamp(genStamp);
     doReturn(completedBlockInfo).when(namesystemSpy)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 1e42e34..f654107 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -69,7 +69,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -205,7 +205,7 @@ public class TestEditLog {
 
       for (int i = 0; i < numTransactions; i++) {
         INodeFile inode = new INodeFile(namesystem.dir.allocateNewInodeId(), null,
-            p, 0L, 0L, BlockInfoContiguous.EMPTY_ARRAY, replication, blockSize);
+            p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
         inode.toUnderConstruction("", "");
 
         editLog.logOpenFile("/filename" + (startIndex + i), inode, false, false);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 5653df5..27a1bd3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
@@ -105,7 +105,7 @@ public class TestFSImage {
       INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile();
       assertEquals("hello".length(), file2Node.computeFileSize());
       assertTrue(file2Node.isUnderConstruction());
-      BlockInfoContiguous[] blks = file2Node.getBlocks();
+      BlockInfo[] blks = file2Node.getBlocks();
       assertEquals(1, blks.length);
       assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
       // check lease manager

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 7f31f84..eabd0c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -79,7 +79,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
@@ -885,7 +885,7 @@ public class TestFsck {
       // intentionally corrupt NN data structure
       INodeFile node = (INodeFile) cluster.getNamesystem().dir.getINode
           (fileName, true);
-      final BlockInfoContiguous[] blocks = node.getBlocks();
+      final BlockInfo[] blocks = node.getBlocks();
       assertEquals(blocks.length, 1);
       blocks[0].setNumBytes(-1L);  // set the block length to be negative
       

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
index 4d0f994..7a3a8d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
@@ -21,7 +21,7 @@ import org.apache.commons.io.Charsets;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.junit.Test;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
@@ -125,7 +125,7 @@ public class TestGetBlockLocations {
         FsPermission.createImmutable((short) 0x1ff));
     final INodeFile file = new INodeFile(
         MOCK_INODE_ID, FILE_NAME.getBytes(Charsets.UTF_8),
-        perm, 1, 1, new BlockInfoContiguous[] {}, (short) 1,
+        perm, 1, 1, new BlockInfo[] {}, (short) 1,
         DFS_BLOCK_SIZE_DEFAULT);
     fsn.getFSDirectory().addINode(iip, file);
     return fsn;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index b45d2f6..21b67a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -60,7 +60,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
@@ -290,7 +290,7 @@ public class TestINodeFile {
       iNodes[i] = new INodeFile(i, null, perm, 0L, 0L, null, replication,
           preferredBlockSize);
       iNodes[i].setLocalName(DFSUtil.string2Bytes(fileNamePrefix + i));
-      BlockInfoContiguous newblock = new BlockInfoContiguous(replication);
+      BlockInfo newblock = new BlockInfo(replication);
       iNodes[i].addBlock(newblock);
     }
     


[10/50] [abbrv] hadoop git commit: HDFS-5033. Bad error message for fs -put/copyFromLocal if user doesn't have permissions to read the source (Darrell Taylor via aw)

Posted by zj...@apache.org.
HDFS-5033. Bad error message for fs -put/copyFromLocal if user doesn't have permissions to read the source (Darrell Taylor via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fdef78a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fdef78a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fdef78a

Branch: refs/heads/YARN-2928
Commit: 6fdef78a65c8c7e6fdf1bdfb05706fd867fc28e4
Parents: e06811e
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed May 27 16:51:34 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:53 2015 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/fs/FileUtil.java     | 158 +++++-----
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   3 +
 .../org/apache/hadoop/hdfs/TestDFSShell.java    | 297 ++++++++++++-------
 3 files changed, 268 insertions(+), 190 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fdef78a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 91f00e1..5fd89c4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -65,7 +65,7 @@ public class FileUtil {
 
   /**
    * convert an array of FileStatus to an array of Path
-   * 
+   *
    * @param stats
    *          an array of FileStatus objects
    * @return an array of paths corresponding to the input
@@ -95,7 +95,7 @@ public class FileUtil {
     else
       return stat2Paths(stats);
   }
-  
+
   /**
    * Delete a directory and all its contents.  If
    * we return false, the directory may be partially-deleted.
@@ -110,7 +110,7 @@ public class FileUtil {
   public static boolean fullyDelete(final File dir) {
     return fullyDelete(dir, false);
   }
-  
+
   /**
    * Delete a directory and all its contents.  If
    * we return false, the directory may be partially-deleted.
@@ -127,7 +127,7 @@ public class FileUtil {
    */
   public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
     if (tryGrantPermissions) {
-      // try to chmod +rwx the parent folder of the 'dir': 
+      // try to chmod +rwx the parent folder of the 'dir':
       File parent = dir.getParentFile();
       grantPermissions(parent);
     }
@@ -189,7 +189,7 @@ public class FileUtil {
     }
     return !ex;
   }
-  
+
   /**
    * Delete the contents of a directory, not the directory itself.  If
    * we return false, the directory may be partially-deleted.
@@ -199,19 +199,19 @@ public class FileUtil {
   public static boolean fullyDeleteContents(final File dir) {
     return fullyDeleteContents(dir, false);
   }
-  
+
   /**
    * Delete the contents of a directory, not the directory itself.  If
    * we return false, the directory may be partially-deleted.
    * If dir is a symlink to a directory, all the contents of the actual
    * directory pointed to by dir will be deleted.
-   * @param tryGrantPermissions if 'true', try grant +rwx permissions to this 
+   * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
    * and all the underlying directories before trying to delete their contents.
    */
   public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
     if (tryGrantPermissions) {
       // to be able to list the dir and delete files from it
-      // we must grant the dir rwx permissions: 
+      // we must grant the dir rwx permissions:
       grantPermissions(dir);
     }
     boolean deletionSucceeded = true;
@@ -246,14 +246,14 @@ public class FileUtil {
 
   /**
    * Recursively delete a directory.
-   * 
+   *
    * @param fs {@link FileSystem} on which the path is present
-   * @param dir directory to recursively delete 
+   * @param dir directory to recursively delete
    * @throws IOException
    * @deprecated Use {@link FileSystem#delete(Path, boolean)}
    */
   @Deprecated
-  public static void fullyDelete(FileSystem fs, Path dir) 
+  public static void fullyDelete(FileSystem fs, Path dir)
   throws IOException {
     fs.delete(dir, true);
   }
@@ -262,9 +262,9 @@ public class FileUtil {
   // If the destination is a subdirectory of the source, then
   // generate exception
   //
-  private static void checkDependencies(FileSystem srcFS, 
-                                        Path src, 
-                                        FileSystem dstFS, 
+  private static void checkDependencies(FileSystem srcFS,
+                                        Path src,
+                                        FileSystem dstFS,
                                         Path dst)
                                         throws IOException {
     if (srcFS == dstFS) {
@@ -282,16 +282,16 @@ public class FileUtil {
   }
 
   /** Copy files between FileSystems. */
-  public static boolean copy(FileSystem srcFS, Path src, 
-                             FileSystem dstFS, Path dst, 
+  public static boolean copy(FileSystem srcFS, Path src,
+                             FileSystem dstFS, Path dst,
                              boolean deleteSource,
                              Configuration conf) throws IOException {
     return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
   }
 
-  public static boolean copy(FileSystem srcFS, Path[] srcs, 
+  public static boolean copy(FileSystem srcFS, Path[] srcs,
                              FileSystem dstFS, Path dst,
-                             boolean deleteSource, 
+                             boolean deleteSource,
                              boolean overwrite, Configuration conf)
                              throws IOException {
     boolean gotException = false;
@@ -307,7 +307,7 @@ public class FileUtil {
                             "does not exist");
     } else {
       FileStatus sdst = dstFS.getFileStatus(dst);
-      if (!sdst.isDirectory()) 
+      if (!sdst.isDirectory())
         throw new IOException("copying multiple files, but last argument `" +
                               dst + "' is not a directory");
     }
@@ -329,8 +329,8 @@ public class FileUtil {
   }
 
   /** Copy files between FileSystems. */
-  public static boolean copy(FileSystem srcFS, Path src, 
-                             FileSystem dstFS, Path dst, 
+  public static boolean copy(FileSystem srcFS, Path src,
+                             FileSystem dstFS, Path dst,
                              boolean deleteSource,
                              boolean overwrite,
                              Configuration conf) throws IOException {
@@ -375,21 +375,21 @@ public class FileUtil {
     } else {
       return true;
     }
-  
+
   }
 
   /** Copy all files in a directory to one output file (merge). */
-  public static boolean copyMerge(FileSystem srcFS, Path srcDir, 
-                                  FileSystem dstFS, Path dstFile, 
+  public static boolean copyMerge(FileSystem srcFS, Path srcDir,
+                                  FileSystem dstFS, Path dstFile,
                                   boolean deleteSource,
                                   Configuration conf, String addString) throws IOException {
     dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
 
     if (!srcFS.getFileStatus(srcDir).isDirectory())
       return false;
-   
+
     OutputStream out = dstFS.create(dstFile);
-    
+
     try {
       FileStatus contents[] = srcFS.listStatus(srcDir);
       Arrays.sort(contents);
@@ -400,24 +400,24 @@ public class FileUtil {
             IOUtils.copyBytes(in, out, conf, false);
             if (addString!=null)
               out.write(addString.getBytes("UTF-8"));
-                
+
           } finally {
             in.close();
-          } 
+          }
         }
       }
     } finally {
       out.close();
     }
-    
+
 
     if (deleteSource) {
       return srcFS.delete(srcDir, true);
     } else {
       return true;
     }
-  }  
-  
+  }
+
   /** Copy local files to a FileSystem. */
   public static boolean copy(File src,
                              FileSystem dstFS, Path dst,
@@ -446,8 +446,12 @@ public class FileUtil {
         IOUtils.closeStream( in );
         throw e;
       }
+    } else if (!src.canRead()) {
+      throw new IOException(src.toString() +
+                            ": Permission denied");
+
     } else {
-      throw new IOException(src.toString() + 
+      throw new IOException(src.toString() +
                             ": No such file or directory");
     }
     if (deleteSource) {
@@ -458,7 +462,7 @@ public class FileUtil {
   }
 
   /** Copy FileSystem files to local files. */
-  public static boolean copy(FileSystem srcFS, Path src, 
+  public static boolean copy(FileSystem srcFS, Path src,
                              File dst, boolean deleteSource,
                              Configuration conf) throws IOException {
     FileStatus filestatus = srcFS.getFileStatus(src);
@@ -516,7 +520,7 @@ public class FileUtil {
   public static String makeShellPath(String filename) throws IOException {
     return filename;
   }
-  
+
   /**
    * Convert a os-native filename to a path that works for the shell.
    * @param file The filename to convert
@@ -530,12 +534,12 @@ public class FileUtil {
   /**
    * Convert a os-native filename to a path that works for the shell.
    * @param file The filename to convert
-   * @param makeCanonicalPath 
+   * @param makeCanonicalPath
    *          Whether to make canonical path for the file passed
    * @return The unix pathname
    * @throws IOException on windows, there can be problems with the subprocess
    */
-  public static String makeShellPath(File file, boolean makeCanonicalPath) 
+  public static String makeShellPath(File file, boolean makeCanonicalPath)
   throws IOException {
     if (makeCanonicalPath) {
       return makeShellPath(file.getCanonicalPath());
@@ -547,7 +551,7 @@ public class FileUtil {
   /**
    * Takes an input dir and returns the du on that local directory. Very basic
    * implementation.
-   * 
+   *
    * @param dir
    *          The input dir to get the disk space of this local dir
    * @return The total disk space of the input local directory
@@ -576,7 +580,7 @@ public class FileUtil {
       return size;
     }
   }
-    
+
   /**
    * Given a File input it will unzip the file in a the unzip directory
    * passed as the second parameter
@@ -596,9 +600,9 @@ public class FileUtil {
           InputStream in = zipFile.getInputStream(entry);
           try {
             File file = new File(unzipDir, entry.getName());
-            if (!file.getParentFile().mkdirs()) {           
+            if (!file.getParentFile().mkdirs()) {
               if (!file.getParentFile().isDirectory()) {
-                throw new IOException("Mkdirs failed to create " + 
+                throw new IOException("Mkdirs failed to create " +
                                       file.getParentFile().toString());
               }
             }
@@ -625,10 +629,10 @@ public class FileUtil {
   /**
    * Given a Tar File as input it will untar the file in a the untar directory
    * passed as the second parameter
-   * 
+   *
    * This utility will untar ".tar" files and ".tar.gz","tgz" files.
-   *  
-   * @param inFile The tar file as input. 
+   *
+   * @param inFile The tar file as input.
    * @param untarDir The untar directory where to untar the tar file.
    * @throws IOException
    */
@@ -641,17 +645,17 @@ public class FileUtil {
 
     boolean gzipped = inFile.toString().endsWith("gz");
     if(Shell.WINDOWS) {
-      // Tar is not native to Windows. Use simple Java based implementation for 
+      // Tar is not native to Windows. Use simple Java based implementation for
       // tests and simple tar archives
       unTarUsingJava(inFile, untarDir, gzipped);
     }
     else {
-      // spawn tar utility to untar archive for full fledged unix behavior such 
+      // spawn tar utility to untar archive for full fledged unix behavior such
       // as resolving symlinks in tar archives
       unTarUsingTar(inFile, untarDir, gzipped);
     }
   }
-  
+
   private static void unTarUsingTar(File inFile, File untarDir,
       boolean gzipped) throws IOException {
     StringBuffer untarCommand = new StringBuffer();
@@ -659,9 +663,9 @@ public class FileUtil {
       untarCommand.append(" gzip -dc '");
       untarCommand.append(FileUtil.makeShellPath(inFile));
       untarCommand.append("' | (");
-    } 
+    }
     untarCommand.append("cd '");
-    untarCommand.append(FileUtil.makeShellPath(untarDir)); 
+    untarCommand.append(FileUtil.makeShellPath(untarDir));
     untarCommand.append("' ; ");
     untarCommand.append("tar -xf ");
 
@@ -675,11 +679,11 @@ public class FileUtil {
     shexec.execute();
     int exitcode = shexec.getExitCode();
     if (exitcode != 0) {
-      throw new IOException("Error untarring file " + inFile + 
+      throw new IOException("Error untarring file " + inFile +
                   ". Tar process exited with exit code " + exitcode);
     }
   }
-  
+
   private static void unTarUsingJava(File inFile, File untarDir,
       boolean gzipped) throws IOException {
     InputStream inputStream = null;
@@ -702,7 +706,7 @@ public class FileUtil {
       IOUtils.cleanup(LOG, tis, inputStream);
     }
   }
-  
+
   private static void unpackEntries(TarArchiveInputStream tis,
       TarArchiveEntry entry, File outputDir) throws IOException {
     if (entry.isDirectory()) {
@@ -739,14 +743,14 @@ public class FileUtil {
     outputStream.flush();
     outputStream.close();
   }
-  
+
   /**
    * Class for creating hardlinks.
    * Supports Unix, WindXP.
    * @deprecated Use {@link org.apache.hadoop.fs.HardLink}
    */
   @Deprecated
-  public static class HardLink extends org.apache.hadoop.fs.HardLink { 
+  public static class HardLink extends org.apache.hadoop.fs.HardLink {
     // This is a stub to assist with coordinated change between
     // COMMON and HDFS projects.  It will be removed after the
     // corresponding change is committed to HDFS.
@@ -759,7 +763,7 @@ public class FileUtil {
    * setting, we will log a warning. The return code in this
    * case is 2.
    *
-   * @param target the target for symlink 
+   * @param target the target for symlink
    * @param linkname the symlink
    * @return 0 on success
    */
@@ -873,7 +877,7 @@ public class FileUtil {
       shExec.execute();
     }catch(IOException e) {
       if(LOG.isDebugEnabled()) {
-        LOG.debug("Error while changing permission : " + filename 
+        LOG.debug("Error while changing permission : " + filename
                   +" Exception: " + StringUtils.stringifyException(e));
       }
     }
@@ -1041,9 +1045,9 @@ public class FileUtil {
       execSetPermission(f, permission);
       return;
     }
-    
+
     boolean rv = true;
-    
+
     // read perms
     rv = f.setReadable(group.implies(FsAction.READ), false);
     checkReturnValue(rv, f, permission);
@@ -1069,17 +1073,17 @@ public class FileUtil {
     }
   }
 
-  private static void checkReturnValue(boolean rv, File p, 
+  private static void checkReturnValue(boolean rv, File p,
                                        FsPermission permission
                                        ) throws IOException {
     if (!rv) {
-      throw new IOException("Failed to set permissions of path: " + p + 
-                            " to " + 
+      throw new IOException("Failed to set permissions of path: " + p +
+                            " to " +
                             String.format("%04o", permission.toShort()));
     }
   }
-  
-  private static void execSetPermission(File f, 
+
+  private static void execSetPermission(File f,
                                         FsPermission permission
                                        )  throws IOException {
     if (NativeIO.isAvailable()) {
@@ -1089,7 +1093,7 @@ public class FileUtil {
                   String.format("%04o", permission.toShort()), false));
     }
   }
-  
+
   static String execCommand(File f, String... cmd) throws IOException {
     String[] args = new String[cmd.length + 1];
     System.arraycopy(cmd, 0, args, 0, cmd.length);
@@ -1147,12 +1151,12 @@ public class FileUtil {
       }
     }
   }
-  
+
   /**
-   * A wrapper for {@link File#listFiles()}. This java.io API returns null 
+   * A wrapper for {@link File#listFiles()}. This java.io API returns null
    * when a dir is not a directory or for any I/O error. Instead of having
    * null check everywhere File#listFiles() is used, we will add utility API
-   * to get around this problem. For the majority of cases where we prefer 
+   * to get around this problem. For the majority of cases where we prefer
    * an IOException to be thrown.
    * @param dir directory for which listing should be performed
    * @return list of files or empty list
@@ -1165,13 +1169,13 @@ public class FileUtil {
                 + dir.toString());
     }
     return files;
-  }  
-  
+  }
+
   /**
-   * A wrapper for {@link File#list()}. This java.io API returns null 
+   * A wrapper for {@link File#list()}. This java.io API returns null
    * when a dir is not a directory or for any I/O error. Instead of having
    * null check everywhere File#list() is used, we will add utility API
-   * to get around this problem. For the majority of cases where we prefer 
+   * to get around this problem. For the majority of cases where we prefer
    * an IOException to be thrown.
    * @param dir directory for which listing should be performed
    * @return list of file names or empty string list
@@ -1184,35 +1188,35 @@ public class FileUtil {
                 + dir.toString());
     }
     return fileNames;
-  }  
-  
+  }
+
   public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
       Map<String, String> callerEnv) throws IOException {
     return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv);
   }
-  
+
   /**
    * Create a jar file at the given path, containing a manifest with a classpath
    * that references all specified entries.
-   * 
+   *
    * Some platforms may have an upper limit on command line length.  For example,
    * the maximum command line length on Windows is 8191 characters, but the
    * length of the classpath may exceed this.  To work around this limitation,
    * use this method to create a small intermediate jar with a manifest that
    * contains the full classpath.  It returns the absolute path to the new jar,
    * which the caller may set as the classpath for a new process.
-   * 
+   *
    * Environment variable evaluation is not supported within a jar manifest, so
    * this method expands environment variables before inserting classpath entries
    * to the manifest.  The method parses environment variables according to
    * platform-specific syntax (%VAR% on Windows, or $VAR otherwise).  On Windows,
    * environment variables are case-insensitive.  For example, %VAR% and %var%
    * evaluate to the same value.
-   * 
+   *
    * Specifying the classpath in a jar manifest does not support wildcards, so
    * this method expands wildcards internally.  Any classpath entry that ends
    * with * is translated to all files at that path with extension .jar or .JAR.
-   * 
+   *
    * @param inputClassPath String input classpath to bundle into the jar manifest
    * @param pwd Path to working directory to save jar
    * @param targetDir path to where the jar execution will have its working dir

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fdef78a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 948a516..439b53a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -160,6 +160,9 @@ Trunk (Unreleased)
 
     HDFS-6353. Check and make checkpoint before stopping the NameNode. (jing9)
 
+    HDFS-5033. Bad error message for fs -put/copyFromLocal if user
+    doesn't have permissions to read the source (Darrell Taylor via aw)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fdef78a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index c1bf771..2df31c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -155,13 +155,13 @@ public class TestDFSShell {
       assertTrue(f1.exists());
       assertTrue(f1.isFile());
       assertEquals(0L, f1.length());
-      
+
       //copy to remote
       final Path root = mkdir(dfs, new Path("/test/zeroSizeFile"));
       final Path remotef = new Path(root, "dst");
       show("copy local " + f1 + " to remote " + remotef);
       dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), remotef);
-      
+
       //getBlockSize() should not throw exception
       show("Block size = " + dfs.getFileStatus(remotef).getBlockSize());
 
@@ -172,7 +172,7 @@ public class TestDFSShell {
       assertTrue(f2.exists());
       assertTrue(f2.isFile());
       assertEquals(0L, f2.length());
-  
+
       f1.delete();
       f2.delete();
     } finally {
@@ -180,13 +180,13 @@ public class TestDFSShell {
       cluster.shutdown();
     }
   }
-  
+
   @Test (timeout = 30000)
   public void testRecursiveRm() throws IOException {
 	  Configuration conf = new HdfsConfiguration();
 	  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
 	  FileSystem fs = cluster.getFileSystem();
-	  assertTrue("Not a HDFS: " + fs.getUri(), 
+	  assertTrue("Not a HDFS: " + fs.getUri(),
 			  fs instanceof DistributedFileSystem);
 	  try {
       fs.mkdirs(new Path(new Path("parent"), "child"));
@@ -201,12 +201,12 @@ public class TestDFSShell {
       } catch(IOException e) {
         assert(false);
       }
-    } finally {  
+    } finally {
       try { fs.close();}catch(IOException e){};
       cluster.shutdown();
     }
   }
-    
+
   @Test (timeout = 30000)
   public void testDu() throws IOException {
     int replication = 2;
@@ -220,7 +220,7 @@ public class TestDFSShell {
     System.setOut(psOut);
     FsShell shell = new FsShell();
     shell.setConf(conf);
-    
+
     try {
       Path myPath = new Path("/test/dir");
       assertTrue(fs.mkdirs(myPath));
@@ -235,7 +235,7 @@ public class TestDFSShell {
       Long myFileDiskUsed = myFileLength * replication;
       Long myFile2Length = fs.getFileStatus(myFile2).getLen();
       Long myFile2DiskUsed = myFile2Length * replication;
-      
+
       String[] args = new String[2];
       args[0] = "-du";
       args[1] = "/test/dir";
@@ -254,7 +254,7 @@ public class TestDFSShell {
       assertThat(returnString, containsString(myFileDiskUsed.toString()));
       assertThat(returnString, containsString(myFile2Length.toString()));
       assertThat(returnString, containsString(myFile2DiskUsed.toString()));
-      
+
       // Check that -du -s reports the state of the snapshot
       String snapshotName = "ss1";
       Path snapshotPath = new Path(myPath, ".snapshot/" + snapshotName);
@@ -306,7 +306,7 @@ public class TestDFSShell {
       System.setOut(psBackup);
       cluster.shutdown();
     }
-                                  
+
   }
 
   @Test (timeout = 30000)
@@ -321,15 +321,15 @@ public class TestDFSShell {
     try {
       // remove left over crc files:
       new File(TEST_ROOT_DIR, ".f1.crc").delete();
-      new File(TEST_ROOT_DIR, ".f2.crc").delete();    
+      new File(TEST_ROOT_DIR, ".f2.crc").delete();
       final File f1 = createLocalFile(new File(TEST_ROOT_DIR, "f1"));
       final File f2 = createLocalFile(new File(TEST_ROOT_DIR, "f2"));
-  
+
       final Path root = mkdir(dfs, new Path("/test/put"));
       final Path dst = new Path(root, "dst");
-  
+
       show("begin");
-      
+
       final Thread copy2ndFileThread = new Thread() {
         @Override
         public void run() {
@@ -344,13 +344,13 @@ public class TestDFSShell {
           assertTrue(false);
         }
       };
-      
+
       //use SecurityManager to pause the copying of f1 and begin copying f2
       SecurityManager sm = System.getSecurityManager();
       System.out.println("SecurityManager = " + sm);
       System.setSecurityManager(new SecurityManager() {
         private boolean firstTime = true;
-  
+
         @Override
         public void checkPermission(Permission perm) {
           if (firstTime) {
@@ -359,7 +359,7 @@ public class TestDFSShell {
               String s = "" + Arrays.asList(t.getStackTrace());
               if (s.contains("FileUtil.copyContent")) {
                 //pause at FileUtil.copyContent
-  
+
                 firstTime = false;
                 copy2ndFileThread.start();
                 try {Thread.sleep(5000);} catch (InterruptedException e) {}
@@ -371,7 +371,7 @@ public class TestDFSShell {
       show("copy local " + f1 + " to remote " + dst);
       dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), dst);
       show("done");
-  
+
       try {copy2ndFileThread.join();} catch (InterruptedException e) { }
       System.setSecurityManager(sm);
 
@@ -381,8 +381,8 @@ public class TestDFSShell {
       srcs[0] = new Path(f1.getPath());
       srcs[1] = new Path(f2.getPath());
       dfs.copyFromLocalFile(false, false, srcs, destmultiple);
-      srcs[0] = new Path(destmultiple,"f1"); 
-      srcs[1] = new Path(destmultiple,"f2"); 
+      srcs[0] = new Path(destmultiple,"f1");
+      srcs[1] = new Path(destmultiple,"f2");
       assertTrue(dfs.exists(srcs[0]));
       assertTrue(dfs.exists(srcs[1]));
 
@@ -492,7 +492,7 @@ public class TestDFSShell {
       ret = ToolRunner.run(shell, argv);
       returned = out.toString();
       assertEquals(" -mkdir returned 1 ", 1, ret);
-      assertTrue(" -mkdir returned File exists", 
+      assertTrue(" -mkdir returned File exists",
           (returned.lastIndexOf("File exists") != -1));
       Path testFile = new Path("/testfile");
       OutputStream outtmp = srcFs.create(testFile);
@@ -520,7 +520,7 @@ public class TestDFSShell {
       argv[2] = "/testfiletest";
       ret = ToolRunner.run(shell, argv);
       returned = out.toString();
-      assertTrue("no output from rename", 
+      assertTrue("no output from rename",
           (returned.lastIndexOf("Renamed") == -1));
       out.reset();
       argv[0] = "-mv";
@@ -557,7 +557,7 @@ public class TestDFSShell {
       }
     }
   }
-  
+
   @Test (timeout = 30000)
   public void testURIPaths() throws Exception {
     Configuration srcConf = new HdfsConfiguration();
@@ -580,14 +580,14 @@ public class TestDFSShell {
       argv[1] = dstFs.getUri().toString() + "/";
       int ret = ToolRunner.run(shell, argv);
       assertEquals("ls works on remote uri ", 0, ret);
-      //check for rm -r 
+      //check for rm -r
       dstFs.mkdirs(new Path("/hadoopdir"));
       argv = new String[2];
       argv[0] = "-rmr";
       argv[1] = dstFs.getUri().toString() + "/hadoopdir";
       ret = ToolRunner.run(shell, argv);
       assertEquals("-rmr works on remote uri " + argv[1], 0, ret);
-      //check du 
+      //check du
       argv[0] = "-du";
       argv[1] = dstFs.getUri().toString() + "/";
       ret = ToolRunner.run(shell, argv);
@@ -601,14 +601,14 @@ public class TestDFSShell {
       argv[2] = dstFs.getUri().toString() + "/furi";
       ret = ToolRunner.run(shell, argv);
       assertEquals(" put is working ", 0, ret);
-      //check cp 
+      //check cp
       argv[0] = "-cp";
       argv[1] = dstFs.getUri().toString() + "/furi";
       argv[2] = srcFs.getUri().toString() + "/furi";
       ret = ToolRunner.run(shell, argv);
       assertEquals(" cp is working ", 0, ret);
       assertTrue(srcFs.exists(new Path("/furi")));
-      //check cat 
+      //check cat
       argv = new String[2];
       argv[0] = "-cat";
       argv[1] = dstFs.getUri().toString() + "/furi";
@@ -626,7 +626,7 @@ public class TestDFSShell {
       confirmOwner(null, "herbivores", dstFs, parent, path);
       runCmd(shell, "-chown", "-R", ":reptiles", dstFs.getUri().toString() + "/");
       confirmOwner(null, "reptiles", dstFs, root, parent, path);
-      //check if default hdfs:/// works 
+      //check if default hdfs:/// works
       argv[0] = "-cat";
       argv[1] = "hdfs:///furi";
       ret = ToolRunner.run(shell, argv);
@@ -757,7 +757,7 @@ public class TestDFSShell {
       assertTrue("Output doesn't match input",
           Arrays.equals(writebytes, out.toByteArray()));
       out.reset();
-      
+
       // Test a plain text.
       OutputStream pout = fs.create(new Path(root, "file.txt"));
       writebytes = "bar".getBytes();
@@ -805,8 +805,8 @@ public class TestDFSShell {
         }
 
         File localroot = new File(TEST_ROOT_DIR, "copyToLocal");
-        File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2");        
-        
+        File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2");
+
         File f1 = new File(localroot, "f1");
         assertTrue("Copying failed.", f1.isFile());
 
@@ -821,9 +821,9 @@ public class TestDFSShell {
 
         File f4 = new File(sub, "f4");
         assertTrue("Copying failed.", f4.isFile());
-        
+
         File f5 = new File(localroot2, "f1");
-        assertTrue("Copying failed.", f5.isFile());        
+        assertTrue("Copying failed.", f5.isFile());
 
         f1.delete();
         f2.delete();
@@ -836,12 +836,12 @@ public class TestDFSShell {
       // destination files
       {
         String[] args = {"-copyToLocal", "nosuchfile", TEST_ROOT_DIR};
-        try {   
+        try {
           assertEquals(1, shell.run(args));
         } catch (Exception e) {
           System.err.println("Exception raised from DFSShell.run " +
                             e.getLocalizedMessage());
-        }                            
+        }
         File f6 = new File(TEST_ROOT_DIR, "nosuchfile");
         assertTrue(!f6.exists());
       }
@@ -867,7 +867,7 @@ public class TestDFSShell {
     String path = "/test/" + name;
     Path root = mkdir(fs, new Path(path));
     Path sub = mkdir(fs, new Path(root, "sub"));
-    Path root2 = mkdir(fs, new Path(path + "2"));        
+    Path root2 = mkdir(fs, new Path(path + "2"));
 
     writeFile(fs, new Path(root, "f1"));
     writeFile(fs, new Path(root, "f2"));
@@ -900,7 +900,7 @@ public class TestDFSShell {
       localpath = localpath.makeQualified(localfs.getUri(),
           localfs.getWorkingDirectory());
       localfs.mkdirs(localpath);
-      
+
       final String localstr = localpath.toString();
       System.out.println("localstr=" + localstr);
       runCount(localstr, 1, 0, shell);
@@ -915,7 +915,7 @@ public class TestDFSShell {
   }
   private static void runCount(String path, long dirs, long files, FsShell shell
     ) throws IOException {
-    ByteArrayOutputStream bytes = new ByteArrayOutputStream(); 
+    ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     PrintStream out = new PrintStream(bytes);
     PrintStream oldOut = System.out;
     System.setOut(out);
@@ -956,15 +956,15 @@ public class TestDFSShell {
       throw new IOException(StringUtils.stringifyException(e));
     }
   }
-  
+
   /**
    * Test chmod.
    */
-  void testChmod(Configuration conf, FileSystem fs, String chmodDir) 
+  void testChmod(Configuration conf, FileSystem fs, String chmodDir)
                                                     throws IOException {
     FsShell shell = new FsShell();
     shell.setConf(conf);
-    
+
     try {
       //first make dir
       Path dir = new Path(chmodDir);
@@ -1039,8 +1039,8 @@ public class TestDFSShell {
     LOG.info("Permission change result: " + result);
     assertEquals(expected, result);
   }
-   
-  private void confirmOwner(String owner, String group, 
+
+  private void confirmOwner(String owner, String group,
                             FileSystem fs, Path... paths) throws IOException {
     for(Path path : paths) {
       if (owner != null) {
@@ -1051,68 +1051,68 @@ public class TestDFSShell {
       }
     }
   }
-  
+
   @Test (timeout = 30000)
   public void testFilePermissions() throws IOException {
     Configuration conf = new HdfsConfiguration();
-    
+
     //test chmod on local fs
     FileSystem fs = FileSystem.getLocal(conf);
-    testChmod(conf, fs, 
+    testChmod(conf, fs,
               (new File(TEST_ROOT_DIR, "chmodTest")).getAbsolutePath());
-    
+
     conf.set(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, "true");
-    
+
     //test chmod on DFS
     MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
     fs = cluster.getFileSystem();
     testChmod(conf, fs, "/tmp/chmodTest");
-    
+
     // test chown and chgrp on DFS:
-    
+
     FsShell shell = new FsShell();
     shell.setConf(conf);
     fs = cluster.getFileSystem();
-    
+
     /* For dfs, I am the super user and I can change owner of any file to
      * anything. "-R" option is already tested by chmod test above.
      */
-    
+
     String file = "/tmp/chownTest";
     Path path = new Path(file);
     Path parent = new Path("/tmp");
     Path root = new Path("/");
     TestDFSShell.writeFile(fs, path);
-    
+
     runCmd(shell, "-chgrp", "-R", "herbivores", "/*", "unknownFile*");
     confirmOwner(null, "herbivores", fs, parent, path);
-    
+
     runCmd(shell, "-chgrp", "mammals", file);
     confirmOwner(null, "mammals", fs, path);
-    
+
     runCmd(shell, "-chown", "-R", ":reptiles", "/");
     confirmOwner(null, "reptiles", fs, root, parent, path);
-    
+
     runCmd(shell, "-chown", "python:", "/nonExistentFile", file);
     confirmOwner("python", "reptiles", fs, path);
 
     runCmd(shell, "-chown", "-R", "hadoop:toys", "unknownFile", "/");
     confirmOwner("hadoop", "toys", fs, root, parent, path);
-    
+
     // Test different characters in names
 
     runCmd(shell, "-chown", "hdfs.user", file);
     confirmOwner("hdfs.user", null, fs, path);
-    
+
     runCmd(shell, "-chown", "_Hdfs.User-10:_hadoop.users--", file);
     confirmOwner("_Hdfs.User-10", "_hadoop.users--", fs, path);
-    
+
     runCmd(shell, "-chown", "hdfs/hadoop-core@apache.org:asf-projects", file);
     confirmOwner("hdfs/hadoop-core@apache.org", "asf-projects", fs, path);
-    
+
     runCmd(shell, "-chgrp", "hadoop-core@apache.org/100", file);
     confirmOwner(null, "hadoop-core@apache.org/100", fs, path);
-    
+
     cluster.shutdown();
   }
   /**
@@ -1142,7 +1142,7 @@ public class TestDFSShell {
       Path myFile = new Path("/test/mkdirs/myFile");
       writeFile(fileSys, myFile);
       assertTrue(fileSys.exists(myFile));
-      Path myFile2 = new Path("/test/mkdirs/myFile2");      
+      Path myFile2 = new Path("/test/mkdirs/myFile2");
       writeFile(fileSys, myFile2);
       assertTrue(fileSys.exists(myFile2));
 
@@ -1156,7 +1156,7 @@ public class TestDFSShell {
           val = shell.run(args);
         } catch (Exception e) {
           System.err.println("Exception raised from DFSShell.run " +
-                             e.getLocalizedMessage()); 
+                             e.getLocalizedMessage());
         }
         assertTrue(val == 0);
         assertFalse(fileSys.exists(myFile));
@@ -1174,13 +1174,13 @@ public class TestDFSShell {
         String[] args = new String[3];
         args[0] = "-cat";
         args[1] = "/test/mkdirs/myFile";
-        args[2] = "/test/mkdirs/myFile2";        
+        args[2] = "/test/mkdirs/myFile2";
         int val = -1;
         try {
           val = shell.run(args);
         } catch (Exception e) {
           System.err.println("Exception raised from DFSShell.run: " +
-                             StringUtils.stringifyException(e)); 
+                             StringUtils.stringifyException(e));
         }
         assertTrue(val == 0);
       }
@@ -1196,7 +1196,7 @@ public class TestDFSShell {
           val = shell.run(args);
         } catch (Exception e) {
           System.err.println("Exception raised from DFSShell.run " +
-                             e.getLocalizedMessage()); 
+                             e.getLocalizedMessage());
         }
         assertTrue(val != 0);
       }
@@ -1211,7 +1211,7 @@ public class TestDFSShell {
           val = shell.run(args);
         } catch (Exception e) {
           System.err.println("Exception raised from DFSShell.run " +
-                             e.getLocalizedMessage()); 
+                             e.getLocalizedMessage());
         }
         assertTrue(val != 0);
       }
@@ -1226,7 +1226,7 @@ public class TestDFSShell {
           val = shell.run(args);
         } catch (Exception e) {
           System.err.println("Exception raised from DFSShell.run " +
-                             e.getLocalizedMessage()); 
+                             e.getLocalizedMessage());
         }
         assertTrue(val == 0);
       }
@@ -1480,7 +1480,7 @@ public class TestDFSShell {
         for(Block b : e.getValue()) {
           files.add(DataNodeTestUtils.getFile(dn, poolId, b.getBlockId()));
         }
-      }        
+      }
     }
     return files;
   }
@@ -1493,7 +1493,7 @@ public class TestDFSShell {
       PrintWriter out = new PrintWriter(f);
       out.print(content);
       out.flush();
-      out.close();      
+      out.close();
     }
   }
 
@@ -1503,7 +1503,7 @@ public class TestDFSShell {
 
   @Test (timeout = 30000)
   public void testRemoteException() throws Exception {
-    UserGroupInformation tmpUGI = 
+    UserGroupInformation tmpUGI =
       UserGroupInformation.createUserForTesting("tmpname", new String[] {"mygroup"});
     MiniDFSCluster dfs = null;
     PrintStream bak = null;
@@ -1515,7 +1515,7 @@ public class TestDFSShell {
       fs.mkdirs(p);
       fs.setPermission(p, new FsPermission((short)0700));
       bak = System.err;
-      
+
       tmpUGI.doAs(new PrivilegedExceptionAction<Object>() {
         @Override
         public Object run() throws Exception {
@@ -1529,9 +1529,9 @@ public class TestDFSShell {
           int ret = ToolRunner.run(fshell, args);
           assertEquals("returned should be 1", 1, ret);
           String str = out.toString();
-          assertTrue("permission denied printed", 
+          assertTrue("permission denied printed",
                      str.indexOf("Permission denied") != -1);
-          out.reset();           
+          out.reset();
           return null;
         }
       });
@@ -1544,7 +1544,7 @@ public class TestDFSShell {
       }
     }
   }
-  
+
   @Test (timeout = 30000)
   public void testGet() throws IOException {
     GenericTestUtils.setLogLevel(FSInputChecker.LOG, Level.ALL);
@@ -1563,7 +1563,7 @@ public class TestDFSShell {
     	  String dst = new File(TEST_ROOT_DIR, fname + ++count)
             .getAbsolutePath();
     	  String[] args = new String[options.length + 3];
-    	  args[0] = "-get"; 
+    	  args[0] = "-get";
     	  args[args.length - 2] = remotef.toString();
     	  args[args.length - 1] = dst;
     	  for(int i = 0; i < options.length; i++) {
@@ -1574,9 +1574,9 @@ public class TestDFSShell {
     	  try {
     	    assertEquals(exitcode, shell.run(args));
     	  } catch (Exception e) {
-    	    assertTrue(StringUtils.stringifyException(e), false); 
+    	    assertTrue(StringUtils.stringifyException(e), false);
     	  }
-    	  return exitcode == 0? DFSTestUtil.readFile(new File(dst)): null; 
+    	  return exitcode == 0? DFSTestUtil.readFile(new File(dst)): null;
     	}
     };
 
@@ -1645,9 +1645,9 @@ public class TestDFSShell {
     try {
       final String root = createTree(dfs, "lsr");
       dfs.mkdirs(new Path(root, "zzz"));
-      
+
       runLsr(new FsShell(conf), root, 0);
-      
+
       final Path sub = new Path(root, "sub");
       dfs.setPermission(sub, new FsPermission((short)0));
 
@@ -1670,7 +1670,7 @@ public class TestDFSShell {
   private static String runLsr(final FsShell shell, String root, int returnvalue
       ) throws Exception {
     System.out.println("root=" + root + ", returnvalue=" + returnvalue);
-    final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); 
+    final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     final PrintStream oldErr = System.err;
@@ -1688,7 +1688,7 @@ public class TestDFSShell {
     System.out.println("results:\n" + results);
     return results;
   }
-  
+
   /**
    * default setting is file:// which is not a DFS
    * so DFSAdmin should throw and catch InvalidArgumentException
@@ -1703,7 +1703,7 @@ public class TestDFSShell {
     int res = admin.run(new String[] {"-refreshNodes"});
     assertEquals("expected to fail -1", res , -1);
   }
-  
+
   // Preserve Copy Option is -ptopxa (timestamps, ownership, permission, XATTR,
   // ACLs)
   @Test (timeout = 120000)
@@ -1737,15 +1737,15 @@ public class TestDFSShell {
       final String owner = status.getOwner();
       final String group = status.getGroup();
       final FsPermission perm = status.getPermission();
-      
+
       fs.setXAttr(src, USER_A1, USER_A1_VALUE);
       fs.setXAttr(src, TRUSTED_A1, TRUSTED_A1_VALUE);
-      
+
       shell = new FsShell(conf);
-      
+
       // -p
       Path target1 = new Path(hdfsTestDir, "targetfile1");
-      String[] argv = new String[] { "-cp", "-p", src.toUri().toString(), 
+      String[] argv = new String[] { "-cp", "-p", src.toUri().toString(),
           target1.toUri().toString() };
       int ret = ToolRunner.run(shell, argv);
       assertEquals("cp -p is not working", SUCCESS, ret);
@@ -1764,7 +1764,7 @@ public class TestDFSShell {
 
       // -ptop
       Path target2 = new Path(hdfsTestDir, "targetfile2");
-      argv = new String[] { "-cp", "-ptop", src.toUri().toString(), 
+      argv = new String[] { "-cp", "-ptop", src.toUri().toString(),
           target2.toUri().toString() };
       ret = ToolRunner.run(shell, argv);
       assertEquals("cp -ptop is not working", SUCCESS, ret);
@@ -1783,7 +1783,7 @@ public class TestDFSShell {
 
       // -ptopx
       Path target3 = new Path(hdfsTestDir, "targetfile3");
-      argv = new String[] { "-cp", "-ptopx", src.toUri().toString(), 
+      argv = new String[] { "-cp", "-ptopx", src.toUri().toString(),
           target3.toUri().toString() };
       ret = ToolRunner.run(shell, argv);
       assertEquals("cp -ptopx is not working", SUCCESS, ret);
@@ -2317,6 +2317,77 @@ public class TestDFSShell {
     }
   }
 
+  /* [refs HDFS-5033]
+   *
+   * return a "Permission Denied" message instead of "No such file or Directory"
+   * when trying to put/copyFromLocal a file that doesn't have read access
+   *
+   */
+  @Test (timeout = 30000)
+  public void testCopyFromLocalWithPermissionDenied() throws Exception {
+    Configuration conf = new Configuration();
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+        .format(true).build();
+    FsShell shell = null;
+    FileSystem fs = null;
+    PrintStream bak = null;
+
+    final File localFile = new File(TEST_ROOT_DIR, "testFileWithNoReadPermissions");
+    final String localfilepath = new Path(localFile.getAbsolutePath()).toUri().toString();
+    final String testdir = "/tmp/TestDFSShell-CopyFromLocalWithPermissionDenied-"
+        + counter.getAndIncrement();
+    final Path hdfsTestDir = new Path(testdir);
+    try {
+      fs = cluster.getFileSystem();
+      fs.mkdirs(hdfsTestDir);
+      localFile.createNewFile();
+      localFile.setReadable(false);
+      writeFile(fs, new Path(testdir, "testFileForPut"));
+      shell = new FsShell();
+
+      // capture system error messages, snarfed from testErrOutPut()
+      bak = System.err;
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      PrintStream tmp = new PrintStream(out);
+      System.setErr(tmp);
+
+      // Tests for put
+      String[] argv = new String[] { "-put", localfilepath, testdir };
+      int res = ToolRunner.run(shell, argv);
+      assertEquals("put is working", ERROR, res);
+      String returned = out.toString();
+      assertTrue(" outputs Permission denied error message",
+          (returned.lastIndexOf("Permission denied") != -1));
+
+      // Tests for copyFromLocal
+      argv = new String[] { "-copyFromLocal", localfilepath, testdir };
+      res = ToolRunner.run(shell, argv);
+      assertEquals("copyFromLocal -f is working", ERROR, res);
+      returned = out.toString();
+      assertTrue(" outputs Permission denied error message",
+          (returned.lastIndexOf("Permission denied") != -1));
+
+    } finally {
+      if (bak != null) {
+        System.setErr(bak);
+      }
+
+      if (null != shell)
+        shell.close();
+
+      if (localFile.exists())
+        localFile.delete();
+
+      if (null != fs) {
+        fs.delete(hdfsTestDir, true);
+        fs.close();
+      }
+      cluster.shutdown();
+    }
+  }
+
+
+
   // setrep for file and directory.
   @Test (timeout = 30000)
   public void testSetrep() throws Exception {
@@ -2506,7 +2577,7 @@ public class TestDFSShell {
       cluster.shutdown();
     }
   }
-  
+
   @Test (timeout = 30000)
   public void testSetXAttrPermission() throws Exception {
     UserGroupInformation user = UserGroupInformation.
@@ -2517,16 +2588,16 @@ public class TestDFSShell {
       final Configuration conf = new HdfsConfiguration();
       cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
       cluster.waitActive();
-      
+
       FileSystem fs = cluster.getFileSystem();
       Path p = new Path("/foo");
       fs.mkdirs(p);
       bak = System.err;
-      
+
       final FsShell fshell = new FsShell(conf);
       final ByteArrayOutputStream out = new ByteArrayOutputStream();
       System.setErr(new PrintStream(out));
-      
+
       // No permission to write xattr
       fs.setPermission(p, new FsPermission((short) 0700));
       user.doAs(new PrivilegedExceptionAction<Object>() {
@@ -2536,18 +2607,18 @@ public class TestDFSShell {
               "-setfattr", "-n", "user.a1", "-v", "1234", "/foo"});
           assertEquals("Returned should be 1", 1, ret);
           String str = out.toString();
-          assertTrue("Permission denied printed", 
+          assertTrue("Permission denied printed",
               str.indexOf("Permission denied") != -1);
           out.reset();
           return null;
         }
       });
-      
+
       int ret = ToolRunner.run(fshell, new String[]{
           "-setfattr", "-n", "user.a1", "-v", "1234", "/foo"});
       assertEquals("Returned should be 0", 0, ret);
       out.reset();
-      
+
       // No permission to read and remove
       fs.setPermission(p, new FsPermission((short) 0750));
       user.doAs(new PrivilegedExceptionAction<Object>() {
@@ -2560,7 +2631,7 @@ public class TestDFSShell {
           String str = out.toString();
           assertTrue("Permission denied printed",
               str.indexOf("Permission denied") != -1);
-          out.reset();           
+          out.reset();
           // Remove
           ret = ToolRunner.run(fshell, new String[]{
               "-setfattr", "-x", "user.a1", "/foo"});
@@ -2568,7 +2639,7 @@ public class TestDFSShell {
           str = out.toString();
           assertTrue("Permission denied printed",
               str.indexOf("Permission denied") != -1);
-          out.reset();  
+          out.reset();
           return null;
         }
       });
@@ -2683,28 +2754,28 @@ public class TestDFSShell {
   }
 
   /**
-   * 
+   *
    * Test to make sure that user namespace xattrs can be set only if path has
    * access and for sticky directorries, only owner/privileged user can write.
    * Trusted namespace xattrs can be set only with privileged users.
-   * 
+   *
    * As user1: Create a directory (/foo) as user1, chown it to user1 (and
    * user1's group), grant rwx to "other".
-   * 
+   *
    * As user2: Set an xattr (should pass with path access).
-   * 
+   *
    * As user1: Set an xattr (should pass).
-   * 
+   *
    * As user2: Read the xattr (should pass). Remove the xattr (should pass with
    * path access).
-   * 
+   *
    * As user1: Read the xattr (should pass). Remove the xattr (should pass).
-   * 
+   *
    * As user1: Change permissions only to owner
-   * 
+   *
    * As User2: Set an Xattr (Should fail set with no path access) Remove an
    * Xattr (Should fail with no path access)
-   * 
+   *
    * As SuperUser: Set an Xattr with Trusted (Should pass)
    */
   @Test (timeout = 30000)
@@ -2742,7 +2813,7 @@ public class TestDFSShell {
           return null;
         }
       });
-      
+
       //Test 2. Give access to others
       user1.doAs(new PrivilegedExceptionAction<Object>() {
         @Override
@@ -2809,7 +2880,7 @@ public class TestDFSShell {
           return null;
         }
       });
-      
+
       // Test 7. Change permission to have path access only to owner(user1)
       user1.doAs(new PrivilegedExceptionAction<Object>() {
         @Override
@@ -2822,7 +2893,7 @@ public class TestDFSShell {
           return null;
         }
       });
-      
+
       // Test 8. There should be no permissions to set for
       // the non-owning user with no path access.
       user2.doAs(new PrivilegedExceptionAction<Object>() {
@@ -2839,7 +2910,7 @@ public class TestDFSShell {
           return null;
         }
       });
-      
+
       // Test 9. There should be no permissions to remove for
       // the non-owning user with no path access.
       user2.doAs(new PrivilegedExceptionAction<Object>() {
@@ -2856,7 +2927,7 @@ public class TestDFSShell {
           return null;
         }
       });
-      
+
       // Test 10. Superuser should be allowed to set with trusted namespace
       SUPERUSER.doAs(new PrivilegedExceptionAction<Object>() {
         @Override


[06/50] [abbrv] hadoop git commit: YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var for a container (Darrell Taylor via aw)

Posted by zj...@apache.org.
YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var for a container (Darrell Taylor via aw)

Conflicts:
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e06811e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e06811e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e06811e6

Branch: refs/heads/YARN-2928
Commit: e06811e689447cb30418ae66f1e39fe9ff2ecbb3
Parents: 0cd01a1
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed May 27 16:40:56 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:32 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../hadoop/yarn/api/ApplicationConstants.java   | 50 +++++++++-----------
 .../resourcemanager/amlauncher/AMLauncher.java  | 29 +++++-------
 .../TestApplicationMasterLauncher.java          |  7 ++-
 4 files changed, 41 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c8dbb08..54b33bc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -136,6 +136,9 @@ Trunk - Unreleased
     YARN-2428. LCE default banned user list should have yarn (Varun
     Saxena via aw)
 
+    YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var
+    for a container (Darrell Taylor via aw)
+
 Release 2.8.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index 42464da..d75297a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.util.Shell;
 /**
  * This is the API for the applications comprising of constants that YARN sets
  * up for the applications and the containers.
- * 
+ *
  * TODO: Investigate the semantics and security of each cross-boundary refs.
  */
 @Public
 @Evolving
 public interface ApplicationConstants {
-  
+
   /**
    * The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
    * only
@@ -48,11 +48,11 @@ public interface ApplicationConstants {
       UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
 
   /**
-   * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in 
+   * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
    * ApplicationMaster's environment only. This states that for all non-relative
    * web URLs in the app masters web UI what base should they have.
    */
-  public static final String APPLICATION_WEB_PROXY_BASE_ENV = 
+  public static final String APPLICATION_WEB_PROXY_BASE_ENV =
     "APPLICATION_WEB_PROXY_BASE";
 
   /**
@@ -99,15 +99,9 @@ public interface ApplicationConstants {
   public static final String STDOUT = "stdout";
 
   /**
-   * The environment variable for MAX_APP_ATTEMPTS. Set in AppMaster environment
-   * only
-   */
-  public static final String MAX_APP_ATTEMPTS_ENV = "MAX_APP_ATTEMPTS";
-
-  /**
    * Environment for Applications.
-   * 
-   * Some of the environment variables for applications are <em>final</em> 
+   *
+   * Some of the environment variables for applications are <em>final</em>
    * i.e. they cannot be modified by the applications.
    */
   public enum Environment {
@@ -116,76 +110,76 @@ public interface ApplicationConstants {
      * Final, non-modifiable.
      */
     USER("USER"),
-    
+
     /**
      * $LOGNAME
      * Final, non-modifiable.
      */
     LOGNAME("LOGNAME"),
-    
+
     /**
      * $HOME
      * Final, non-modifiable.
      */
     HOME("HOME"),
-    
+
     /**
      * $PWD
      * Final, non-modifiable.
      */
     PWD("PWD"),
-    
+
     /**
      * $PATH
      */
     PATH("PATH"),
-    
+
     /**
      * $SHELL
      */
     SHELL("SHELL"),
-    
+
     /**
      * $JAVA_HOME
      */
     JAVA_HOME("JAVA_HOME"),
-    
+
     /**
      * $CLASSPATH
      */
     CLASSPATH("CLASSPATH"),
-    
+
     /**
      * $APP_CLASSPATH
      */
     APP_CLASSPATH("APP_CLASSPATH"),
-    
+
     /**
      * $LD_LIBRARY_PATH
      */
     LD_LIBRARY_PATH("LD_LIBRARY_PATH"),
-    
+
     /**
      * $HADOOP_CONF_DIR
      * Final, non-modifiable.
      */
     HADOOP_CONF_DIR("HADOOP_CONF_DIR"),
-    
+
     /**
      * $HADOOP_COMMON_HOME
      */
     HADOOP_COMMON_HOME("HADOOP_COMMON_HOME"),
-    
+
     /**
      * $HADOOP_HDFS_HOME
      */
     HADOOP_HDFS_HOME("HADOOP_HDFS_HOME"),
-    
+
     /**
      * $MALLOC_ARENA_MAX
      */
     MALLOC_ARENA_MAX("MALLOC_ARENA_MAX"),
-    
+
     /**
      * $HADOOP_YARN_HOME
      */
@@ -240,11 +234,11 @@ public interface ApplicationConstants {
     private Environment(String variable) {
       this.variable = variable;
     }
-    
+
     public String key() {
       return variable;
     }
-    
+
     public String toString() {
       return variable;
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index f62ee50..42bf579 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -80,10 +80,10 @@ public class AMLauncher implements Runnable {
   private final AMLauncherEventType eventType;
   private final RMContext rmContext;
   private final Container masterContainer;
-  
+
   @SuppressWarnings("rawtypes")
   private final EventHandler handler;
-  
+
   public AMLauncher(RMContext rmContext, RMAppAttempt application,
       AMLauncherEventType eventType, Configuration conf) {
     this.application = application;
@@ -93,20 +93,20 @@ public class AMLauncher implements Runnable {
     this.handler = rmContext.getDispatcher().getEventHandler();
     this.masterContainer = application.getMasterContainer();
   }
-  
+
   private void connect() throws IOException {
     ContainerId masterContainerID = masterContainer.getId();
-    
+
     containerMgrProxy = getContainerMgrProxy(masterContainerID);
   }
-  
+
   private void launch() throws IOException, YarnException {
     connect();
     ContainerId masterContainerID = masterContainer.getId();
     ApplicationSubmissionContext applicationContext =
       application.getSubmissionContext();
     LOG.info("Setting up container " + masterContainer
-        + " for AM " + application.getAppAttemptId());  
+        + " for AM " + application.getAppAttemptId());
     ContainerLaunchContext launchContext =
         createAMContainerLaunchContext(applicationContext, masterContainerID);
 
@@ -130,7 +130,7 @@ public class AMLauncher implements Runnable {
           + application.getAppAttemptId());
     }
   }
-  
+
   private void cleanup() throws IOException, YarnException {
     connect();
     ContainerId containerId = masterContainer.getId();
@@ -188,17 +188,17 @@ public class AMLauncher implements Runnable {
       ContainerId containerID) throws IOException {
 
     // Construct the actual Container
-    ContainerLaunchContext container = 
+    ContainerLaunchContext container =
         applicationMasterContext.getAMContainerSpec();
     LOG.info("Command to launch container "
         + containerID
         + " : "
         + StringUtils.arrayToString(container.getCommands().toArray(
             new String[0])));
-    
+
     // Finalize the container
     setupTokens(container, containerID);
-    
+
     return container;
   }
 
@@ -210,7 +210,7 @@ public class AMLauncher implements Runnable {
     Map<String, String> environment = container.getEnvironment();
     environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV,
         application.getWebProxyBase());
-    // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
+    // Set AppSubmitTime to be consumable by the AM.
     ApplicationId applicationId =
         application.getAppAttemptId().getApplicationId();
     environment.put(
@@ -218,9 +218,6 @@ public class AMLauncher implements Runnable {
         String.valueOf(rmContext.getRMApps()
             .get(applicationId)
             .getSubmitTime()));
-    environment.put(ApplicationConstants.MAX_APP_ATTEMPTS_ENV,
-        String.valueOf(rmContext.getRMApps().get(
-            applicationId).getMaxAppAttempts()));
     // Set flow context info
     for (String tag :
         rmContext.getRMApps().get(applicationId).getApplicationTags()) {
@@ -267,7 +264,7 @@ public class AMLauncher implements Runnable {
     ((RMAppAttemptImpl)application).setAMRMToken(amrmToken);
     return amrmToken;
   }
-  
+
   @SuppressWarnings("unchecked")
   public void run() {
     switch (eventType) {
@@ -297,7 +294,7 @@ public class AMLauncher implements Runnable {
         sb.append(" is not handled by this NodeManager");
         if (!e.getMessage().contains(sb.toString())) {
           // Ignoring if container is already killed by Node Manager.
-          LOG.info("Error cleaning master ", e);          
+          LOG.info("Error cleaning master ", e);
         }
       }
       break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
index 9a4395e..d4f8e93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
@@ -107,8 +107,7 @@ public class TestApplicationMasterLauncher {
       nmHostAtContainerManager = tokenId.getNmHostAddress();
       submitTimeAtContainerManager =
           Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
-      maxAppAttempts =
-          Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
+      maxAppAttempts = YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS;
       return StartContainersResponse.newInstance(
         new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
         new HashMap<ContainerId, SerializedException>());
@@ -184,8 +183,8 @@ public class TestApplicationMasterLauncher {
     am.waitForState(RMAppAttemptState.FINISHED);
     rm.stop();
   }
-  
-    
+
+
   @SuppressWarnings("unused")
   @Test(timeout = 100000)
   public void testallocateBeforeAMRegistration() throws Exception {


[34/50] [abbrv] hadoop git commit: YARN-3713. Remove duplicate function call storeContainerDiagnostics in ContainerDiagnosticsUpdateTransition (zxu via rkanter)

Posted by zj...@apache.org.
YARN-3713. Remove duplicate function call storeContainerDiagnostics in ContainerDiagnosticsUpdateTransition (zxu via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/704eee9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/704eee9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/704eee9e

Branch: refs/heads/YARN-2928
Commit: 704eee9e8b13968bbdbab2020ef445c508f5c239
Parents: 6b126e9
Author: Robert Kanter <rk...@apache.org>
Authored: Fri May 29 15:34:37 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:58 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                               | 3 +++
 .../nodemanager/containermanager/container/ContainerImpl.java | 7 -------
 2 files changed, 3 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/704eee9e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4ddf11b..906d9d4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -368,6 +368,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3722. Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils.
     (Masatake Iwasaki via devaraj)
 
+    YARN-3713. Remove duplicate function call storeContainerDiagnostics in
+    ContainerDiagnosticsUpdateTransition (zxu via rkanter)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

http://git-wip-us.apache.org/repos/asf/hadoop/blob/704eee9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 9997ca2..289d6d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1105,13 +1105,6 @@ public class ContainerImpl implements Container {
       ContainerDiagnosticsUpdateEvent updateEvent =
           (ContainerDiagnosticsUpdateEvent) event;
       container.addDiagnostics(updateEvent.getDiagnosticsUpdate(), "\n");
-      try {
-        container.stateStore.storeContainerDiagnostics(container.containerId,
-            container.diagnostics);
-      } catch (IOException e) {
-        LOG.warn("Unable to update state store diagnostics for "
-            + container.containerId, e);
-      }
     }
   }
 


[42/50] [abbrv] hadoop git commit: HDFS-8490. Typo in trace enabled log in ExceptionHandler of WebHDFS. Contributed by Archana T.

Posted by zj...@apache.org.
HDFS-8490. Typo in trace enabled log in ExceptionHandler of WebHDFS. Contributed by Archana T.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bc36be7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bc36be7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bc36be7

Branch: refs/heads/YARN-2928
Commit: 8bc36be7aa76b5f214854a49f14d2470719e606b
Parents: 00ccf73
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Mon Jun 1 00:41:50 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:13:00 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt                       | 3 +++
 .../hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bc36be7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cc8235c..10e5da4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -825,6 +825,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-7609. Avoid retry cache collision when Standby NameNode loading edits.
     (Ming Ma via jing9)
 
+    HDFS-8490. Typo in trace enabled log in ExceptionHandler of WebHDFS.
+    (Archana T via ozawa)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bc36be7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
index 779a397..a6a8aa1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
@@ -50,7 +50,7 @@ class ExceptionHandler {
     Exception e = cause instanceof Exception ? (Exception) cause : new Exception(cause);
 
     if (LOG.isTraceEnabled()) {
-      LOG.trace("GOT EXCEPITION", e);
+      LOG.trace("GOT EXCEPTION", e);
     }
 
     //Convert exception


[45/50] [abbrv] hadoop git commit: YARN-3725. App submission via REST API is broken in secure mode due to Timeline DT service address is empty. (Zhijie Shen via wangda)

Posted by zj...@apache.org.
YARN-3725. App submission via REST API is broken in secure mode due to Timeline DT service address is empty. (Zhijie Shen via wangda)

Conflicts:
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/732a70f1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/732a70f1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/732a70f1

Branch: refs/heads/YARN-2928
Commit: 732a70f12f5c9b84486503275bfe60a219aba44c
Parents: a977bed
Author: Wangda Tan <wa...@apache.org>
Authored: Sun May 31 16:30:34 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:04 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../client/api/impl/TimelineClientImpl.java     | 30 ++++++++++++++------
 .../TestTimelineAuthenticationFilter.java       | 11 +++++++
 3 files changed, 36 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/732a70f1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e770a5b..807165f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -697,6 +697,9 @@ Release 2.7.1 - UNRELEASED
     YARN-2900. Application (Attempt and Container) Not Found in AHS results
     in Internal Server Error (500). (Zhijie Shen and Mit Desai via xgong)
 
+    YARN-3725. App submission via REST API is broken in secure mode due to 
+    Timeline DT service address is empty. (Zhijie Shen via wangda)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/732a70f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index e44a8a9..77e9af6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -522,9 +522,12 @@ public class TimelineClientImpl extends TimelineClient {
   public long renewDelegationToken(
       final Token<TimelineDelegationTokenIdentifier> timelineDT)
           throws IOException, YarnException {
-    boolean useHttps = YarnConfiguration.useHttps(this.getConfig());
-    final String scheme = useHttps ? "https" : "http";
-    final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT);
+    final boolean isTokenServiceAddrEmpty =
+        timelineDT.getService().toString().isEmpty();
+    final String scheme = isTokenServiceAddrEmpty ? null
+        : (YarnConfiguration.useHttps(this.getConfig()) ? "https" : "http");
+    final InetSocketAddress address = isTokenServiceAddrEmpty ? null
+        : SecurityUtil.getTokenServiceAddr(timelineDT);
     PrivilegedExceptionAction<Long> renewDTAction =
         new PrivilegedExceptionAction<Long>() {
 
@@ -539,7 +542,11 @@ public class TimelineClientImpl extends TimelineClient {
             DelegationTokenAuthenticatedURL authUrl =
                 new DelegationTokenAuthenticatedURL(authenticator,
                     connConfigurator);
-            final URI serviceURI = new URI(scheme, null, address.getHostName(),
+            // If the token service address is not available, fall back to use
+            // the configured service address.
+            final URI serviceURI = isTokenServiceAddrEmpty ?
+                constructResURI(getConfig(), getTimelineServiceAddress(), false)
+                : new URI(scheme, null, address.getHostName(),
                 address.getPort(), RESOURCE_URI_STR_V1, null, null);
             return authUrl
                 .renewDelegationToken(serviceURI.toURL(), token, doAsUser);
@@ -553,9 +560,12 @@ public class TimelineClientImpl extends TimelineClient {
   public void cancelDelegationToken(
       final Token<TimelineDelegationTokenIdentifier> timelineDT)
           throws IOException, YarnException {
-    boolean useHttps = YarnConfiguration.useHttps(this.getConfig());
-    final String scheme = useHttps ? "https" : "http";
-    final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT);
+    final boolean isTokenServiceAddrEmpty =
+        timelineDT.getService().toString().isEmpty();
+    final String scheme = isTokenServiceAddrEmpty ? null
+        : (YarnConfiguration.useHttps(this.getConfig()) ? "https" : "http");
+    final InetSocketAddress address = isTokenServiceAddrEmpty ? null
+        : SecurityUtil.getTokenServiceAddr(timelineDT);
     PrivilegedExceptionAction<Void> cancelDTAction =
         new PrivilegedExceptionAction<Void>() {
 
@@ -570,7 +580,11 @@ public class TimelineClientImpl extends TimelineClient {
             DelegationTokenAuthenticatedURL authUrl =
                 new DelegationTokenAuthenticatedURL(authenticator,
                     connConfigurator);
-            final URI serviceURI = new URI(scheme, null, address.getHostName(),
+            // If the token service address is not available, fall back to use
+            // the configured service address.
+            final URI serviceURI = isTokenServiceAddrEmpty ?
+                constructResURI(getConfig(), getTimelineServiceAddress(), false)
+                : new URI(scheme, null, address.getHostName(),
                 address.getPort(), RESOURCE_URI_STR_V1, null, null);
             authUrl.cancelDelegationToken(serviceURI.toURL(), token, doAsUser);
             return null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/732a70f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
index c93e8f2..063f512 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
@@ -240,12 +240,21 @@ public class TestTimelineAuthenticationFilter {
     Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner());
 
     // Renew token
+    Assert.assertFalse(token.getService().toString().isEmpty());
+    // Renew the token from the token service address
     long renewTime1 = httpUserClient.renewDelegationToken(token);
     Thread.sleep(100);
+    token.setService(new Text());
+    Assert.assertTrue(token.getService().toString().isEmpty());
+    // If the token service address is not avaiable, it still can be renewed
+    // from the configured address
     long renewTime2 = httpUserClient.renewDelegationToken(token);
     Assert.assertTrue(renewTime1 < renewTime2);
 
     // Cancel token
+    Assert.assertTrue(token.getService().toString().isEmpty());
+    // If the token service address is not avaiable, it still can be canceled
+    // from the configured address
     httpUserClient.cancelDelegationToken(token);
     // Renew should not be successful because the token is canceled
     try {
@@ -280,6 +289,8 @@ public class TestTimelineAuthenticationFilter {
     Assert.assertTrue(renewTime1 < renewTime2);
 
     // Cancel token
+    Assert.assertFalse(tokenToRenew.getService().toString().isEmpty());
+    // Cancel the token from the token service address
     fooUserClient.cancelDelegationToken(tokenToRenew);
 
     // Renew should not be successful because the token is canceled


[13/50] [abbrv] hadoop git commit: HADOOP-12030. test-patch should only report on newly introduced findbugs warnings. (Sean Busbey via aw)

Posted by zj...@apache.org.
HADOOP-12030. test-patch should only report on newly introduced findbugs warnings. (Sean Busbey via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac3000a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac3000a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac3000a2

Branch: refs/heads/YARN-2928
Commit: ac3000a283e10e69635b25b1e25aabc779602ba3
Parents: 3eb698c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 09:53:50 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700

----------------------------------------------------------------------
 dev-support/test-patch.sh                       | 258 +++++++++++++++----
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 2 files changed, 211 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac3000a2/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 5276023..f760951 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -44,6 +44,7 @@ function setup_defaults
   LOAD_SYSTEM_PLUGINS=true
 
   FINDBUGS_HOME=${FINDBUGS_HOME:-}
+  FINDBUGS_WARNINGS_FAIL_PRECHECK=false
   ECLIPSE_HOME=${ECLIPSE_HOME:-}
   BUILD_NATIVE=${BUILD_NATIVE:-true}
   PATCH_BRANCH=""
@@ -585,6 +586,7 @@ function hadoop_usage
   echo "--debug                If set, then output some extra stuff to stderr"
   echo "--dirty-workspace      Allow the local git workspace to have uncommitted changes"
   echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
+  echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
   echo "--issue-re=<expr>      Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
   echo "--modulelist=<list>    Specify additional modules to test (comma delimited)"
   echo "--offline              Avoid connecting to the Internet"
@@ -666,6 +668,9 @@ function parse_args
       --findbugs-home=*)
         FINDBUGS_HOME=${i#*=}
       ;;
+      --findbugs-strict-precheck)
+        FINDBUGS_WARNINGS_FAIL_PRECHECK=true
+      ;;
       --git-cmd=*)
         GIT=${i#*=}
       ;;
@@ -1060,6 +1065,12 @@ function precheck_without_patch
     echo "Patch does not appear to need site tests."
   fi
 
+  precheck_findbugs
+
+  if [[ $? != 0 ]] ; then
+    return 1
+  fi
+
   add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
   return 0
 }
@@ -1838,40 +1849,84 @@ function check_mvn_install
   return ${retval}
 }
 
-## @description  Verify patch does not trigger any findbugs warnings
+## @description  are the needed bits for findbugs present?
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+## @return       0 findbugs will work for our use
+## @return       1 findbugs is missing some component
+function findbugs_is_installed
+{
+  if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
+    printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
+    add_jira_table -1 findbugs "Findbugs is not installed."
+    return 1
+  fi
+  return 0
+}
+
+## @description  Run the maven findbugs plugin and record found issues in a bug database
 ## @audience     private
 ## @stability    evolving
 ## @replaceable  no
 ## @return       0 on success
 ## @return       1 on failure
-function check_findbugs
+function findbugs_mvnrunner
 {
-  local findbugs_version
-  local modules=${CHANGED_MODULES}
-  local rc=0
-  local module_suffix
-  local findbugsWarnings=0
-  local relative_file
-  local newFindbugsWarnings
-  local findbugsWarnings
-  local line
-  local firstpart
-  local secondpart
+  local name=$1
+  local logfile=$2
+  local warnings_file=$3
 
-  big_console_header "Determining number of patched Findbugs warnings."
+  echo_and_redirect "${logfile}" "${MVN}" clean test findbugs:findbugs -DskipTests \
+    "-D${PROJECT_NAME}PatchProcess" < /dev/null
+  if [[ $? != 0 ]]; then
+    return 1
+  fi
+  cp target/findbugsXml.xml "${warnings_file}.xml"
+
+  "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \
+      "${warnings_file}.xml" "${warnings_file}.xml"
+  if [[ $? != 0 ]]; then
+    return 1
+  fi
 
+  "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \
+      "${warnings_file}.html"
+  if [[ $? != 0 ]]; then
+    return 1
+  fi
+
+  return 0
+}
+
+## @description  Track pre-existing findbugs warnings
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+## @return       0 on success
+## @return       1 on failure
+function precheck_findbugs
+{
+  local -r mypwd=$(pwd)
+  local module_suffix
+  local modules=${CHANGED_MODULES}
+  local module
+  local findbugs_version
+  local rc=0
+  local module_findbugs_warnings
+  local findbugs_warnings=0
 
   verify_needed_test findbugs
+
   if [[ $? == 0 ]]; then
-    echo "Patch does not touch any java files. Skipping findbugs."
+    echo "Patch does not appear to need findbugs tests."
     return 0
   fi
 
-  start_clock
+  echo "findbugs baseline for ${mypwd}"
 
-  if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
-    printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
-    add_jira_table -1 findbugs "Findbugs is not installed."
+  findbugs_is_installed
+  if [[ $? != 0 ]]; then
     return 1
   fi
 
@@ -1880,67 +1935,170 @@ function check_findbugs
     pushd "${module}" >/dev/null
     echo "  Running findbugs in ${module}"
     module_suffix=$(basename "${module}")
-    echo_and_redirect "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" "${MVN}" clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess \
-      < /dev/null
+    findbugs_mvnrunner "${PATCH_BRANCH}" \
+        "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \
+        "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}"
     (( rc = rc + $? ))
+
+    if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
+      #shellcheck disable=SC2016
+      module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
+          "${PATCH_BRANCH}" \
+          "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
+          "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
+          | ${AWK} '{print $1}')
+      if [[ $? != 0 ]]; then
+        popd >/dev/null
+        return 1
+      fi
+
+      findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings))
+
+      if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
+        add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html"
+      fi
+    fi
     popd >/dev/null
   done
 
   #shellcheck disable=SC2016
-  findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
+  findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt")
 
   if [[ ${rc} -ne 0 ]]; then
-    add_jira_table -1 findbugs "The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
+    echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?"
+    add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}."
+    return 1
+  fi
+
+  if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \
+        ${findbugs_warnings} -gt 0 ]] ; then
+    echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings."
+    add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings."
+    return 1
+  fi
+
+  return 0
+}
+
+## @description  Verify patch does not trigger any findbugs warnings
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+## @return       0 on success
+## @return       1 on failure
+function check_findbugs
+{
+  local rc=0
+  local module
+  local modules=${CHANGED_MODULES}
+  local module_suffix
+  local combined_xml
+  local newBugs
+  local new_findbugs_warnings
+  local new_findbugs_fixed_warnings
+  local findbugs_warnings=0
+  local findbugs_fixed_warnings=0
+  local line
+  local firstpart
+  local secondpart
+  local findbugs_version
+
+  verify_needed_test findbugs
+
+  if [[ $? == 0 ]]; then
+    return 0
+  fi
+
+  big_console_header "Determining number of patched Findbugs warnings."
+
+  start_clock
+
+  findbugs_is_installed
+  if [[ $? != 0 ]]; then
     return 1
   fi
 
-  while read file
+  for module in ${modules}
   do
-    relative_file=${file#${BASEDIR}/} # strip leading ${BASEDIR} prefix
-    if [[ ${relative_file} != "target/findbugsXml.xml" ]]; then
-      module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
-      module_suffix=$(basename "${module_suffix}")
-    fi
+    pushd "${module}" >/dev/null
+    echo "  Running findbugs in ${module}"
+    module_suffix=$(basename "${module}")
 
-    cp "${file}" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
+    findbugs_mvnrunner patch \
+        "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \
+        "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}"
 
-    "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -timestamp "01/01/2000" \
-      "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \
-      "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
+    if [[ $? != 0 ]] ; then
+      ((rc = rc +1))
+      echo "Post-patch findbugs compilation is broken."
+      add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken."
+      continue
+    fi
 
-    #shellcheck disable=SC2016
-    newFindbugsWarnings=$("${FINDBUGS_HOME}/bin/filterBugs" \
-      -first "01/01/2000" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \
-      "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \
-      | ${AWK} '{print $1}')
+    combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml"
+    newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}"
+    "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
+        -output "${combined_xml}" \
+        "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \
+        "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
+    if [[ $? != 0 ]]; then
+      popd >/dev/null
+      return 1
+    fi
 
-    echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
+    #shellcheck disable=SC2016
+    new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
+        "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
+    if [[ $? != 0 ]]; then
+      popd >/dev/null
+      return 1
+    fi
+    #shellcheck disable=SC2016
+    new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
+        "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
+    if [[ $? != 0 ]]; then
+      popd >/dev/null
+      return 1
+    fi
 
-    findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
+    echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
+    findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
+    findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
 
-    "${FINDBUGS_HOME}/bin/convertXmlToText" -html \
-      "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \
-      "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.html"
+    "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \
+        "${newBugs}.html"
+    if [[ $? != 0 ]]; then
+      popd >/dev/null
+      return 1
+    fi
 
-    if [[ ${newFindbugsWarnings} -gt 0 ]] ; then
+    if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
       populate_test_table FindBugs "module:${module_suffix}"
       while read line; do
         firstpart=$(echo "${line}" | cut -f2 -d:)
         secondpart=$(echo "${line}" | cut -f9- -d' ')
         add_jira_test_table "" "${firstpart}:${secondpart}"
-      done < <("${FINDBUGS_HOME}/bin/convertXmlToText" \
-        "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml")
+      done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml")
 
       add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
     fi
-  done < <(find "${BASEDIR}" -name findbugsXml.xml)
 
-  if [[ ${findbugsWarnings} -gt 0 ]] ; then
-    add_jira_table -1 findbugs "The patch appears to introduce ${findbugsWarnings} new Findbugs (version ${findbugs_version}) warnings."
+    popd >/dev/null
+  done
+
+  #shellcheck disable=SC2016
+  findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
+
+  if [[ ${findbugs_warnings} -gt 0 ]] ; then
+    add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings."
     return 1
   fi
 
-  add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
+  if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then
+    add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings."
+  else
+    add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
+  fi
   return 0
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac3000a2/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1009d1a..eb1db29 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -606,6 +606,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-11594. Improve the readability of site index of documentation.
     (Masatake Iwasaki via aajisaka)
 
+    HADOOP-12030. test-patch should only report on newly introduced
+    findbugs warnings. (Sean Busbey via aw)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp


[15/50] [abbrv] hadoop git commit: HADOOP-12004. test-patch breaks with reexec in certain situations (Sean Busbey via aw)

Posted by zj...@apache.org.
HADOOP-12004. test-patch breaks with reexec in certain situations (Sean Busbey via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dc7948e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dc7948e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dc7948e

Branch: refs/heads/YARN-2928
Commit: 7dc7948e3ca41ee4b54f85f55d7235ee7ed5e85b
Parents: 822cac2
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 08:15:17 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700

----------------------------------------------------------------------
 dev-support/test-patch.sh                       | 2 +-
 hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dc7948e/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index d5d6c26..5276023 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1509,7 +1509,7 @@ function check_reexec
 
   exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \
     --reexec \
-    --branch "${PATCH_BRANCH}" \
+    --branch="${PATCH_BRANCH}" \
     --patch-dir="${PATCH_DIR}" \
       "${USER_PARAMS[@]}"
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dc7948e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d29a885..48e9f7d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -624,6 +624,7 @@ Release 2.8.0 - UNRELEASED
     IPC server. (Masatake Iwasaki via aajisaka)
 
   BUG FIXES
+
     HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there
     is an I/O error during requestShortCircuitShm (cmccabe)
 
@@ -761,6 +762,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-8751. NPE in Token.toString() when Token is constructed using null
     identifier. (kanaka kumar avvaru via aajisaka)
 
+    HADOOP-12004. test-patch breaks with reexec in certain situations (Sean
+    Busbey via aw)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES


[31/50] [abbrv] hadoop git commit: HADOOP-12042. Users may see TrashPolicy if hdfs dfs -rm is run (Contributed by Andreina J)

Posted by zj...@apache.org.
HADOOP-12042. Users may see TrashPolicy if hdfs dfs -rm is run (Contributed by Andreina J)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab02e318
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab02e318
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab02e318

Branch: refs/heads/YARN-2928
Commit: ab02e3187cbabe9b20a730f0d4f7f775f8c1134f
Parents: 0f26da6
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri May 29 16:51:23 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt               | 3 +++
 .../main/java/org/apache/hadoop/fs/TrashPolicyDefault.java    | 7 ++++---
 2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab02e318/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index af82c0b..bc74696 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -797,6 +797,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-11959. WASB should configure client side socket timeout in storage
     client blob request options. (Ivan Mitic via cnauroth)
 
+    HADOOP-12042. Users may see TrashPolicy if hdfs dfs -rm is run
+    (Andreina J via vinayakumarb)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab02e318/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index d6a9b4b..88aeab5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -89,9 +89,6 @@ public class TrashPolicyDefault extends TrashPolicy {
     this.emptierInterval = (long)(conf.getFloat(
         FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT)
         * MSECS_PER_MINUTE);
-    LOG.info("Namenode trash configuration: Deletion interval = " +
-             (this.deletionInterval / MSECS_PER_MINUTE) + " minutes, Emptier interval = " +
-             (this.emptierInterval / MSECS_PER_MINUTE) + " minutes.");
    }
 
   private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
@@ -251,6 +248,10 @@ public class TrashPolicyDefault extends TrashPolicy {
                  " minutes that is used for deletion instead");
         this.emptierInterval = deletionInterval;
       }
+      LOG.info("Namenode trash configuration: Deletion interval = "
+          + (deletionInterval / MSECS_PER_MINUTE)
+          + " minutes, Emptier interval = "
+          + (emptierInterval / MSECS_PER_MINUTE) + " minutes.");
     }
 
     @Override


[37/50] [abbrv] hadoop git commit: HADOOP-12043. Display warning if defaultFs is not set when running fs commands. Contributed by Eddy Xu.

Posted by zj...@apache.org.
HADOOP-12043. Display warning if defaultFs is not set when running fs commands. Contributed by Eddy Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/312030a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/312030a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/312030a3

Branch: refs/heads/YARN-2928
Commit: 312030a3122d10b829c16ab655225128e5434ecb
Parents: 704eee9
Author: Andrew Wang <wa...@apache.org>
Authored: Fri May 29 17:15:58 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:58 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 ++
 .../fs/CommonConfigurationKeysPublic.java       |  6 ++++
 .../org/apache/hadoop/fs/shell/FsCommand.java   | 30 +++++++++++++++++++-
 .../java/org/apache/hadoop/fs/shell/Ls.java     |  8 ++++++
 .../src/main/resources/core-default.xml         |  8 ++++++
 .../java/org/apache/hadoop/fs/shell/TestLs.java | 25 ++++++++++++++++
 6 files changed, 79 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e5d2e39..00075b4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating
     (Masatake Iwasaki via Colin P. McCabe)
 
+    HADOOP-12043. Display warning if defaultFs is not set when running fs
+    commands. (Lei Xu via wang)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 90c6934..7231d59 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -363,5 +363,11 @@ public class CommonConfigurationKeysPublic {
     "hadoop.security.random.device.file.path";
   public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT = 
     "/dev/urandom";
+
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+  public static final String HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY =
+      "hadoop.shell.missing.defaultFs.warning";
+  public static final boolean HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT =
+      false;
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
index 9515fde..971950f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.shell;
 
 import java.io.IOException;
+import java.util.LinkedList;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -27,8 +28,13 @@ import org.apache.hadoop.fs.FsShellPermissions;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.shell.find.Find;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
+
 /**
- * Base class for all "hadoop fs" commands
+ * Base class for all "hadoop fs" commands.
  */
 
 @InterfaceAudience.Private
@@ -90,4 +96,26 @@ abstract public class FsCommand extends Command {
   public int runAll() {
     return run(args);
   }
+
+  @Override
+  protected void processRawArguments(LinkedList<String> args)
+      throws IOException {
+    LinkedList<PathData> expendedArgs = expandArguments(args);
+    // If "fs.defaultFs" is not set appropriately, it warns the user that the
+    // command is not running against HDFS.
+    final boolean displayWarnings = getConf().getBoolean(
+        HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY,
+        HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT);
+    if (displayWarnings) {
+      final String defaultFs = getConf().get(FS_DEFAULT_NAME_KEY);
+      final boolean missingDefaultFs =
+          defaultFs == null || defaultFs.equals(FS_DEFAULT_NAME_DEFAULT);
+      if (missingDefaultFs) {
+        err.printf(
+            "Warning: fs.defaultFs is not set when running \"%s\" command.%n",
+            getCommandName());
+      }
+    }
+    processArguments(expendedArgs);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index d5c52ed..67348c6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Date;
 import java.util.LinkedList;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.StringUtils;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -102,6 +104,12 @@ class Ls extends FsCommand {
 
   protected boolean humanReadable = false;
 
+  protected Ls() {}
+
+  protected Ls(Configuration conf) {
+    super(conf);
+  }
+
   protected String formatSize(long size) {
     return humanReadable
       ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index a1bc780..ae23a75 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1899,4 +1899,12 @@ for ldap providers in the same way as above does.
     <value>Client</value>
   </property>
 
+  <property>
+    <description>
+      Enable hdfs shell commands to display warnings if (fs.defaultFS) property
+      is not set.
+    </description>
+    <name>hadoop.shell.missing.defaultFs.warning</name>
+    <value>false</value>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
index 4a9103f..cba44ee 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
@@ -17,10 +17,12 @@
  */
 package org.apache.hadoop.fs.shell;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
 import static org.junit.Assert.*;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.*;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.net.URI;
@@ -1048,6 +1050,29 @@ public class TestLs {
     verifyNoMoreInteractions(out);
   }
 
+  private static void displayWarningOnLocalFileSystem(boolean shouldDisplay)
+      throws IOException {
+    Configuration conf = new Configuration();
+    conf.setBoolean(
+        HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, shouldDisplay);
+
+    ByteArrayOutputStream buf = new ByteArrayOutputStream();
+    PrintStream err = new PrintStream(buf, true);
+    Ls ls = new Ls(conf);
+    ls.err = err;
+    ls.run("file:///.");
+    assertEquals(shouldDisplay, buf.toString().contains(
+        "Warning: fs.defaultFs is not set when running \"ls\" command."));
+  }
+
+  @Test
+  public void displayWarningsOnLocalFileSystem() throws IOException {
+    // Display warnings.
+    displayWarningOnLocalFileSystem(true);
+    // Does not display warnings.
+    displayWarningOnLocalFileSystem(false);
+  }
+
   // check the deprecated flag isn't set
   @Test
   public void isDeprecated() {


[16/50] [abbrv] hadoop git commit: HDFS-8407. libhdfs hdfsListDirectory must set errno to 0 on success (Masatake Iwasaki via Colin P. McCabe)

Posted by zj...@apache.org.
HDFS-8407. libhdfs hdfsListDirectory must set errno to 0 on success (Masatake Iwasaki via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61510b9d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61510b9d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61510b9d

Branch: refs/heads/YARN-2928
Commit: 61510b9deb0365e250736ccddbe8b4abddc869fa
Parents: b914cca
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu May 28 11:23:44 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt       |  3 +++
 .../hadoop-hdfs/src/main/native/libhdfs/expect.h  | 18 ++++++++++++++++++
 .../hadoop-hdfs/src/main/native/libhdfs/hdfs.c    |  1 +
 .../hadoop-hdfs/src/main/native/libhdfs/hdfs.h    |  3 ++-
 .../main/native/libhdfs/test/test_libhdfs_ops.c   |  9 +++++++++
 .../main/native/libhdfs/test_libhdfs_threaded.c   | 17 ++++++++++++++++-
 6 files changed, 49 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 47481a7..1c9b25e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -813,6 +813,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8431. hdfs crypto class not found in Windows.
     (Anu Engineer via cnauroth)
 
+    HDFS-8407. hdfsListDirectory must set errno to 0 on success (Masatake
+    Iwasaki via Colin P. McCabe)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h
index e64b108..49aa285 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h
@@ -48,6 +48,24 @@ struct hdfsFile_internal;
         } \
     } while (0);
 
+#define EXPECT_NULL_WITH_ERRNO(x, e) \
+    do { \
+        const void* __my_ret__ = x; \
+        int __my_errno__ = errno; \
+        if (__my_ret__ != NULL) { \
+            fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \
+		    "got non-NULL value %p from %s\n", \
+		    __FILE__, __LINE__, __my_errno__, __my_ret__, #x); \
+            return -1; \
+        } \
+        if (__my_errno__ != e) { \
+            fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \
+		    "got expected NULL without expected errno %d from %s\n", \
+		    __FILE__, __LINE__, __my_errno__, e, #x); \
+            return -1; \
+        } \
+    } while (0);
+
 #define EXPECT_NONNULL(x) \
     do { \
         const void* __my_ret__ = x; \

http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
index 504d47e..a3769fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
@@ -3257,6 +3257,7 @@ done:
         return NULL;
     }
     *numEntries = jPathListSize;
+    errno = 0;
     return pathList;
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
index 5b7bc1e..c1515d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
@@ -671,7 +671,8 @@ extern  "C" {
      * @param path The path of the directory. 
      * @param numEntries Set to the number of files/directories in path.
      * @return Returns a dynamically-allocated array of hdfsFileInfo
-     * objects; NULL on error.
+     * objects; NULL on error or empty directory.
+     * errno is set to non-zero on error or zero on success.
      */
     LIBHDFS_EXTERNAL
     hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c
index aa9441a..f564de4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c
@@ -307,6 +307,15 @@ int main(int argc, char **argv) {
         }
 
         fileList = 0;
+        fileList = hdfsListDirectory(fs, newDirectory, &numEntries);
+        if (!(fileList == NULL && numEntries == 0 && !errno)) {
+            fprintf(stderr, "waah! hdfsListDirectory for empty %s - FAILED!\n", newDirectory);
+            totalResult++;
+        } else {
+            fprintf(stderr, "hdfsListDirectory for empty %s - SUCCESS!\n", newDirectory);
+        }
+
+        fileList = 0;
         if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
             for(i=0; i < numEntries; ++i) {
                 fprintf(stderr, "Name: %s, ", fileList[i].mName);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
index 17ff7a8..702430c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
@@ -142,7 +142,7 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
 {
     char tmp[4096];
     hdfsFile file;
-    int ret, expected;
+    int ret, expected, numEntries;
     hdfsFileInfo *fileInfo;
     struct hdfsReadStatistics *readStats = NULL;
 
@@ -153,6 +153,14 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
 
     EXPECT_ZERO(doTestGetDefaultBlockSize(fs, paths->prefix));
 
+    /* There should be no entry in the directory. */
+    errno = EACCES; // see if errno is set to 0 on success
+    EXPECT_NULL_WITH_ERRNO(hdfsListDirectory(fs, paths->prefix, &numEntries), 0);
+    if (numEntries != 0) {
+        fprintf(stderr, "hdfsListDirectory set numEntries to "
+                "%d on empty directory.", numEntries);
+    }
+
     /* There should not be any file to open for reading. */
     EXPECT_NULL(hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0));
 
@@ -179,6 +187,13 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
     EXPECT_ZERO(hdfsHSync(fs, file));
     EXPECT_ZERO(hdfsCloseFile(fs, file));
 
+    /* There should be 1 entry in the directory. */
+    EXPECT_NONNULL(hdfsListDirectory(fs, paths->prefix, &numEntries));
+    if (numEntries != 1) {
+        fprintf(stderr, "hdfsListDirectory set numEntries to "
+                "%d on directory containing 1 file.", numEntries);
+    }
+
     /* Let's re-open the file for reading */
     file = hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0);
     EXPECT_NONNULL(file);


[21/50] [abbrv] hadoop git commit: HADOOP-11142. Remove hdfs dfs reference from file system shell documentation (Kengo Seki via aw)

Posted by zj...@apache.org.
HADOOP-11142. Remove hdfs dfs reference from file system shell documentation (Kengo Seki via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82829c82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82829c82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82829c82

Branch: refs/heads/YARN-2928
Commit: 82829c8293eea24806eb2e44112e5abcd80add29
Parents: 6c65e8a
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 10:57:04 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt                   | 3 +++
 .../hadoop-common/src/site/markdown/FileSystemShell.md            | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/82829c82/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7da02ed..8de5ae4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -221,6 +221,9 @@ Trunk (Unreleased)
     HADOOP-12016. Typo in FileSystem::listStatusIterator 
     (Arthur Vigil via jghoman)
 
+    HADOOP-11142. Remove hdfs dfs reference from file system shell
+    documentation (Kengo Seki via aw)
+
   BUG FIXES
 
     HADOOP-11473. test-patch says "-1 overall" even when all checks are +1

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82829c82/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 8ecc24a..144cb73 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -185,7 +185,7 @@ Example:
 * `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
 * `hadoop fs -count -q hdfs://nn1.example.com/file1`
 * `hadoop fs -count -q -h hdfs://nn1.example.com/file1`
-* `hdfs dfs -count -q -h -v hdfs://nn1.example.com/file1`
+* `hadoop fs -count -q -h -v hdfs://nn1.example.com/file1`
 
 Exit Code:
 


[48/50] [abbrv] hadoop git commit: HDFS-8489. Subclass BlockInfo to represent contiguous blocks. Contributed by Zhe Zhang.

Posted by zj...@apache.org.
HDFS-8489. Subclass BlockInfo to represent contiguous blocks. Contributed by Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/065ab4d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/065ab4d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/065ab4d7

Branch: refs/heads/YARN-2928
Commit: 065ab4d7528df8a099986b6009be7d864d8695bd
Parents: 7505e35
Author: Jing Zhao <ji...@apache.org>
Authored: Mon Jun 1 11:37:47 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:22 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   3 +
 .../hdfs/server/blockmanagement/BlockInfo.java  |  74 +++--------
 .../blockmanagement/BlockInfoContiguous.java    | 123 +++++++++++++++++++
 .../BlockInfoContiguousUnderConstruction.java   |   7 +-
 .../server/blockmanagement/BlockManager.java    |   5 +-
 .../hdfs/server/blockmanagement/BlocksMap.java  |  13 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   3 +-
 .../hdfs/server/namenode/FSImageFormat.java     |   3 +-
 .../server/namenode/FSImageFormatPBINode.java   |   4 +-
 .../server/namenode/FSImageSerialization.java   |   3 +-
 .../snapshot/FSImageFormatPBSnapshot.java       |   3 +-
 .../server/blockmanagement/TestBlockInfo.java   |  14 +--
 .../blockmanagement/TestBlockManager.java       |   4 +-
 .../blockmanagement/TestDatanodeDescriptor.java |   4 +-
 .../blockmanagement/TestPendingReplication.java |   2 +-
 .../blockmanagement/TestReplicationPolicy.java  |   2 +-
 .../hdfs/server/namenode/CreateEditsLog.java    |   5 +-
 .../TestCommitBlockSynchronization.java         |   5 +-
 .../hdfs/server/namenode/TestINodeFile.java     |   3 +-
 .../namenode/TestTruncateQuotaUpdate.java       |   3 +-
 .../snapshot/TestFileWithSnapshotFeature.java   |   3 +-
 21 files changed, 186 insertions(+), 100 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 10e5da4..1f7a36a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -588,6 +588,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8443. Document dfs.namenode.service.handler.count in hdfs-site.xml.
     (J.Andreina via aajisaka)
 
+    HDFS-8489. Subclass BlockInfo to represent contiguous blocks.
+    (Zhe Zhang via jing9)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
index 0370c57..dea31c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.util.LightWeightGSet;
  * the block are stored.
  */
 @InterfaceAudience.Private
-public class BlockInfo extends Block
+public abstract class  BlockInfo extends Block
     implements LightWeightGSet.LinkedElement {
   public static final BlockInfo[] EMPTY_ARRAY = {};
 
@@ -51,7 +51,7 @@ public class BlockInfo extends Block
    * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
    * bytes using the triplets.
    */
-  private Object[] triplets;
+  protected Object[] triplets;
 
   /**
    * Construct an entry for blocksmap
@@ -102,7 +102,7 @@ public class BlockInfo extends Block
     return (DatanodeStorageInfo)triplets[index*3];
   }
 
-  private BlockInfo getPrevious(int index) {
+  BlockInfo getPrevious(int index) {
     assert this.triplets != null : "BlockInfo is not initialized";
     assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
     BlockInfo info = (BlockInfo)triplets[index*3+1];
@@ -122,7 +122,7 @@ public class BlockInfo extends Block
     return info;
   }
 
-  private void setStorageInfo(int index, DatanodeStorageInfo storage) {
+  void setStorageInfo(int index, DatanodeStorageInfo storage) {
     assert this.triplets != null : "BlockInfo is not initialized";
     assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
     triplets[index*3] = storage;
@@ -136,7 +136,7 @@ public class BlockInfo extends Block
    * @param to - block to be set to previous on the list of blocks
    * @return current previous block on the list of blocks
    */
-  private BlockInfo setPrevious(int index, BlockInfo to) {
+  BlockInfo setPrevious(int index, BlockInfo to) {
     assert this.triplets != null : "BlockInfo is not initialized";
     assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
     BlockInfo info = (BlockInfo)triplets[index*3+1];
@@ -152,7 +152,7 @@ public class BlockInfo extends Block
    * @param to - block to be set to next on the list of blocks
    *    * @return current next block on the list of blocks
    */
-  private BlockInfo setNext(int index, BlockInfo to) {
+  BlockInfo setNext(int index, BlockInfo to) {
     assert this.triplets != null : "BlockInfo is not initialized";
     assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
     BlockInfo info = (BlockInfo)triplets[index*3+2];
@@ -167,68 +167,26 @@ public class BlockInfo extends Block
   }
 
   /**
-   * Ensure that there is enough  space to include num more triplets.
-   * @return first free triplet index.
+   * Count the number of data-nodes the block belongs to.
    */
-  private int ensureCapacity(int num) {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    int last = numNodes();
-    if(triplets.length >= (last+num)*3)
-      return last;
-    /* Not enough space left. Create a new array. Should normally
-     * happen only when replication is manually increased by the user. */
-    Object[] old = triplets;
-    triplets = new Object[(last+num)*3];
-    System.arraycopy(old, 0, triplets, 0, last*3);
-    return last;
-  }
+  public abstract int numNodes();
 
   /**
-   * Count the number of data-nodes the block belongs to.
+   * Add a {@link DatanodeStorageInfo} location for a block.
    */
-  public int numNodes() {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    assert triplets.length % 3 == 0 : "Malformed BlockInfo";
-    for(int idx = getCapacity()-1; idx >= 0; idx--) {
-      if(getDatanode(idx) != null)
-        return idx+1;
-    }
-    return 0;
-  }
+  abstract boolean addStorage(DatanodeStorageInfo storage);
 
   /**
-   * Add a {@link DatanodeStorageInfo} location for a block
+   * Remove {@link DatanodeStorageInfo} location for a block
    */
-  boolean addStorage(DatanodeStorageInfo storage) {
-    // find the last null node
-    int lastNode = ensureCapacity(1);
-    setStorageInfo(lastNode, storage);
-    setNext(lastNode, null);
-    setPrevious(lastNode, null);
-    return true;
-  }
+  abstract boolean removeStorage(DatanodeStorageInfo storage);
+
 
   /**
-   * Remove {@link DatanodeStorageInfo} location for a block
+   * Replace the current BlockInfo with the new one in corresponding
+   * DatanodeStorageInfo's linked list
    */
-  boolean removeStorage(DatanodeStorageInfo storage) {
-    int dnIndex = findStorageInfo(storage);
-    if(dnIndex < 0) // the node is not found
-      return false;
-    assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
-      "Block is still in the list and must be removed first.";
-    // find the last not null node
-    int lastNode = numNodes()-1;
-    // replace current node triplet by the lastNode one
-    setStorageInfo(dnIndex, getStorageInfo(lastNode));
-    setNext(dnIndex, getNext(lastNode));
-    setPrevious(dnIndex, getPrevious(lastNode));
-    // set the last triplet to null
-    setStorageInfo(lastNode, null);
-    setNext(lastNode, null);
-    setPrevious(lastNode, null);
-    return true;
-  }
+  abstract void replaceBlock(BlockInfo newBlock);
 
   /**
    * Find specified DatanodeStorageInfo.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
new file mode 100644
index 0000000..eff89a8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+
+/**
+ * Subclass of {@link BlockInfo}, used for a block with replication scheme.
+ */
+@InterfaceAudience.Private
+public class BlockInfoContiguous extends BlockInfo {
+  public static final BlockInfoContiguous[] EMPTY_ARRAY = {};
+
+  public BlockInfoContiguous(short size) {
+    super(size);
+  }
+
+  public BlockInfoContiguous(Block blk, short size) {
+    super(blk, size);
+  }
+
+  /**
+   * Copy construction.
+   * This is used to convert BlockReplicationInfoUnderConstruction
+   * @param from BlockReplicationInfo to copy from.
+   */
+  protected BlockInfoContiguous(BlockInfoContiguous from) {
+    super(from);
+  }
+
+  /**
+   * Ensure that there is enough  space to include num more triplets.
+   * @return first free triplet index.
+   */
+  private int ensureCapacity(int num) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    int last = numNodes();
+    if (triplets.length >= (last+num)*3) {
+      return last;
+    }
+    /* Not enough space left. Create a new array. Should normally
+     * happen only when replication is manually increased by the user. */
+    Object[] old = triplets;
+    triplets = new Object[(last+num)*3];
+    System.arraycopy(old, 0, triplets, 0, last * 3);
+    return last;
+  }
+
+  @Override
+  boolean addStorage(DatanodeStorageInfo storage) {
+    // find the last null node
+    int lastNode = ensureCapacity(1);
+    setStorageInfo(lastNode, storage);
+    setNext(lastNode, null);
+    setPrevious(lastNode, null);
+    return true;
+  }
+
+  @Override
+  boolean removeStorage(DatanodeStorageInfo storage) {
+    int dnIndex = findStorageInfo(storage);
+    if (dnIndex < 0) { // the node is not found
+      return false;
+    }
+    assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+        "Block is still in the list and must be removed first.";
+    // find the last not null node
+    int lastNode = numNodes()-1;
+    // replace current node triplet by the lastNode one
+    setStorageInfo(dnIndex, getStorageInfo(lastNode));
+    setNext(dnIndex, getNext(lastNode));
+    setPrevious(dnIndex, getPrevious(lastNode));
+    // set the last triplet to null
+    setStorageInfo(lastNode, null);
+    setNext(lastNode, null);
+    setPrevious(lastNode, null);
+    return true;
+  }
+
+  @Override
+  public int numNodes() {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+
+    for (int idx = getCapacity()-1; idx >= 0; idx--) {
+      if (getDatanode(idx) != null) {
+        return idx + 1;
+      }
+    }
+    return 0;
+  }
+
+  @Override
+  void replaceBlock(BlockInfo newBlock) {
+    assert newBlock instanceof BlockInfoContiguous;
+    for (int i = this.numNodes() - 1; i >= 0; i--) {
+      final DatanodeStorageInfo storage = this.getStorageInfo(i);
+      final boolean removed = storage.removeBlock(this);
+      assert removed : "currentBlock not found.";
+
+      final DatanodeStorageInfo.AddBlockResult result = storage.addBlock(
+          newBlock);
+      assert result == DatanodeStorageInfo.AddBlockResult.ADDED :
+          "newBlock already exists.";
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
index 58020a6..b757171 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
  * Represents a block that is currently being constructed.<br>
  * This is usually the last block of a file opened for write or append.
  */
-public class BlockInfoContiguousUnderConstruction extends BlockInfo {
+public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
   /** Block state. See {@link BlockUCState} */
   private BlockUCState blockUCState;
 
@@ -165,7 +165,8 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfo {
   /**
    * Create a block that is currently being constructed.
    */
-  public BlockInfoContiguousUnderConstruction(Block blk, short replication, BlockUCState state, DatanodeStorageInfo[] targets) {
+  public BlockInfoContiguousUnderConstruction(Block blk, short replication,
+      BlockUCState state, DatanodeStorageInfo[] targets) {
     super(blk, replication);
     assert getBlockUCState() != BlockUCState.COMPLETE :
       "BlockInfoUnderConstruction cannot be in COMPLETE state";
@@ -185,7 +186,7 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfo {
   BlockInfo convertToCompleteBlock() throws IOException {
     assert getBlockUCState() != BlockUCState.COMPLETE :
       "Trying to convert a COMPLETE block";
-    return new BlockInfo(this);
+    return new BlockInfoContiguous(this);
   }
 
   /** Set expected locations */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 45a485b..d18d7fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1765,7 +1765,8 @@ public class BlockManager {
 
     BlockToMarkCorrupt(BlockInfo stored, long gs, String reason,
         Reason reasonCode) {
-      this(new BlockInfo(stored), stored, reason, reasonCode);
+      this(new BlockInfoContiguous((BlockInfoContiguous)stored), stored,
+          reason, reasonCode);
       //the corrupted block in datanode has a different generation stamp
       corrupted.setGenerationStamp(gs);
     }
@@ -2144,7 +2145,7 @@ public class BlockManager {
 
     // place a delimiter in the list which separates blocks 
     // that have been reported from those that have not
-    BlockInfo delimiter = new BlockInfo(new Block(), (short) 1);
+    BlockInfo delimiter = new BlockInfoContiguous(new Block(), (short) 1);
     AddBlockResult result = storageInfo.addBlock(delimiter);
     assert result == AddBlockResult.ADDED 
         : "Delimiting block cannot be present in the node";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index d06d503..9eb1059 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -20,12 +20,10 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
 import java.util.Iterator;
 
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.util.GSet;
 import org.apache.hadoop.util.LightWeightGSet;
 
-import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
 import com.google.common.collect.Iterables;
 
@@ -222,16 +220,7 @@ class BlocksMap {
     BlockInfo currentBlock = blocks.get(newBlock);
     assert currentBlock != null : "the block if not in blocksMap";
     // replace block in data-node lists
-    for (int i = currentBlock.numNodes() - 1; i >= 0; i--) {
-      final DatanodeDescriptor dn = currentBlock.getDatanode(i);
-      final DatanodeStorageInfo storage = currentBlock.findStorageInfo(dn);
-      final boolean removed = storage.removeBlock(currentBlock);
-      Preconditions.checkState(removed, "currentBlock not found.");
-
-      final AddBlockResult result = storage.addBlock(newBlock);
-      Preconditions.checkState(result == AddBlockResult.ADDED,
-          "newBlock already exists.");
-    }
+    currentBlock.replaceBlock(newBlock);
     // replace block in the map itself
     blocks.put(newBlock);
     return newBlock;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 9acd81f..3de6760 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
@@ -1055,7 +1056,7 @@ public class FSEditLogLoader {
           // is only executed when loading edits written by prior
           // versions of Hadoop. Current versions always log
           // OP_ADD operations as each block is allocated.
-          newBI = new BlockInfo(newBlock,
+          newBI = new BlockInfoContiguous(newBlock,
               file.getPreferredBlockReplication());
         }
         fsNamesys.getBlockManager().addBlockCollection(newBI, file);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index a46f2b6..30517d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.protocol.LayoutFlags;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -755,7 +756,7 @@ public class FSImageFormat {
       // read blocks
       BlockInfo[] blocks = new BlockInfo[numBlocks];
       for (int j = 0; j < numBlocks; j++) {
-        blocks[j] = new BlockInfo(replication);
+        blocks[j] = new BlockInfoContiguous(replication);
         blocks[j].readFields(in);
       }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 29713a2..1c14220 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
@@ -325,7 +326,8 @@ public final class FSImageFormatPBINode {
 
       BlockInfo[] blocks = new BlockInfo[bp.size()];
       for (int i = 0, e = bp.size(); i < e; ++i) {
-        blocks[i] = new BlockInfo(PBHelper.convert(bp.get(i)), replication);
+        blocks[i] =
+            new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
       }
       final PermissionStatus permissions = loadPermission(f.getPermission(),
           parent.getLoaderContext().getStringTable());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index d790b8f..f71cf0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
@@ -131,7 +132,7 @@ public class FSImageSerialization {
     int i = 0;
     for (; i < numBlocks-1; i++) {
       blk.readFields(in);
-      blocks[i] = new BlockInfo(blk, blockReplication);
+      blocks[i] = new BlockInfoContiguous(blk, blockReplication);
     }
     // last block is UNDER_CONSTRUCTION
     if(numBlocks > 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index 363c208..64ad1f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.namenode.AclEntryStatusFormat;
 import org.apache.hadoop.hdfs.server.namenode.AclFeature;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -245,7 +246,7 @@ public class FSImageFormatPBSnapshot {
           BlockInfo storedBlock =  fsn.getBlockManager().getStoredBlock(blk);
           if(storedBlock == null) {
             storedBlock = fsn.getBlockManager().addBlockCollection(
-                new BlockInfo(blk, copy.getFileReplication()), file);
+                new BlockInfoContiguous(blk, copy.getFileReplication()), file);
           }
           blocks[j] = storedBlock;
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index 96d4d8b..5126aa7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -49,7 +49,7 @@ public class TestBlockInfo {
 
   @Test
   public void testIsDeleted() {
-    BlockInfo blockInfo = new BlockInfo((short) 3);
+    BlockInfo blockInfo = new BlockInfoContiguous((short) 3);
     BlockCollection bc = Mockito.mock(BlockCollection.class);
     blockInfo.setBlockCollection(bc);
     Assert.assertFalse(blockInfo.isDeleted());
@@ -59,7 +59,7 @@ public class TestBlockInfo {
 
   @Test
   public void testAddStorage() throws Exception {
-    BlockInfo blockInfo = new BlockInfo((short) 3);
+    BlockInfo blockInfo = new BlockInfoContiguous((short) 3);
 
     final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1");
 
@@ -71,9 +71,9 @@ public class TestBlockInfo {
 
   @Test
   public void testCopyConstructor() {
-    BlockInfo old = new BlockInfo((short) 3);
+    BlockInfo old = new BlockInfoContiguous((short) 3);
     try {
-      BlockInfo copy = new BlockInfo(old);
+      BlockInfo copy = new BlockInfoContiguous((BlockInfoContiguous)old);
       assertEquals(old.getBlockCollection(), copy.getBlockCollection());
       assertEquals(old.getCapacity(), copy.getCapacity());
     } catch (Exception e) {
@@ -92,7 +92,7 @@ public class TestBlockInfo {
 
     // Create a few dummy blocks and add them to the first storage.
     for (int i = 0; i < NUM_BLOCKS; ++i) {
-      blockInfos[i] = new BlockInfo((short) 3);
+      blockInfos[i] = new BlockInfoContiguous((short) 3);
       storage1.addBlock(blockInfos[i]);
     }
 
@@ -118,7 +118,7 @@ public class TestBlockInfo {
     LOG.info("Building block list...");
     for (int i = 0; i < MAX_BLOCKS; i++) {
       blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
-      blockInfoList.add(new BlockInfo(blockList.get(i), (short) 3));
+      blockInfoList.add(new BlockInfoContiguous(blockList.get(i), (short) 3));
       dd.addBlock(blockInfoList.get(i));
 
       // index of the datanode should be 0
@@ -182,4 +182,4 @@ public class TestBlockInfo {
           blockInfoList.get(j), dd.getBlockListHeadForTesting());
     }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index bf46ed7..fd03745 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -389,7 +389,7 @@ public class TestBlockManager {
 
   private BlockInfo blockOnNodes(long blkId, List<DatanodeDescriptor> nodes) {
     Block block = new Block(blkId);
-    BlockInfo blockInfo = new BlockInfo(block, (short) 3);
+    BlockInfo blockInfo = new BlockInfoContiguous(block, (short) 3);
 
     for (DatanodeDescriptor dn : nodes) {
       for (DatanodeStorageInfo storage : dn.getStorageInfos()) {
@@ -738,7 +738,7 @@ public class TestBlockManager {
   private BlockInfo addBlockToBM(long blkId) {
     Block block = new Block(blkId);
     BlockInfo blockInfo =
-        new BlockInfo(block, (short) 3);
+        new BlockInfoContiguous(block, (short) 3);
     BlockCollection bc = Mockito.mock(BlockCollection.class);
     Mockito.doReturn((short) 3).when(bc).getPreferredBlockReplication();
     bm.blocksMap.addBlockCollection(blockInfo, bc);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
index fe639e4..ca27bb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
@@ -58,8 +58,8 @@ public class TestDatanodeDescriptor {
   public void testBlocksCounter() throws Exception {
     DatanodeDescriptor dd = BlockManagerTestUtil.getLocalDatanodeDescriptor(true);
     assertEquals(0, dd.numBlocks());
-    BlockInfo blk = new BlockInfo(new Block(1L), (short) 1);
-    BlockInfo blk1 = new BlockInfo(new Block(2L), (short) 2);
+    BlockInfo blk = new BlockInfoContiguous(new Block(1L), (short) 1);
+    BlockInfo blk1 = new BlockInfoContiguous(new Block(2L), (short) 2);
     DatanodeStorageInfo[] storages = dd.getStorageInfos();
     assertTrue(storages.length > 0);
     // add first block

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
index 844c5d0..090c6e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
@@ -178,7 +178,7 @@ public class TestPendingReplication {
       //
 
       block = new Block(1, 1, 0);
-      blockInfo = new BlockInfo(block, (short) 3);
+      blockInfo = new BlockInfoContiguous(block, (short) 3);
 
       pendingReplications.increment(block,
           DatanodeStorageInfo.toDatanodeDescriptors(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index f317723..fccaf3c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1218,7 +1218,7 @@ public class TestReplicationPolicy {
     chosenBlocks = underReplicatedBlocks.chooseUnderReplicatedBlocks(1);
     assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
 
-    final BlockInfo info = new BlockInfo(block1, (short) 1);
+    final BlockInfo info = new BlockInfoContiguous(block1, (short) 1);
     final BlockCollection mbc = mock(BlockCollection.class);
     when(mbc.getLastBlock()).thenReturn(info);
     when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index 3f96c0c..8a606f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.common.Storage;
 
@@ -69,8 +70,8 @@ public class CreateEditsLog {
     BlockInfo[] blocks = new BlockInfo[blocksPerFile];
     for (int iB = 0; iB < blocksPerFile; ++iB) {
       blocks[iB] = 
-       new BlockInfo(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
-                               replication);
+       new BlockInfoContiguous(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
+           replication);
     }
     
     long currentBlockId = startingBlockId;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index 8c9da01..bea7241 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -106,7 +107,7 @@ public class TestCommitBlockSynchronization {
         lastBlock, genStamp, length, false, false, newTargets, null);
 
     // Simulate 'completing' the block.
-    BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
+    BlockInfo completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
     completedBlockInfo.setBlockCollection(file);
     completedBlockInfo.setGenerationStamp(genStamp);
     doReturn(completedBlockInfo).when(namesystemSpy)
@@ -178,7 +179,7 @@ public class TestCommitBlockSynchronization {
     namesystemSpy.commitBlockSynchronization(
         lastBlock, genStamp, length, true, false, newTargets, null);
 
-    BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
+    BlockInfo completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
     completedBlockInfo.setBlockCollection(file);
     completedBlockInfo.setGenerationStamp(genStamp);
     doReturn(completedBlockInfo).when(namesystemSpy)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index 21b67a7..ec4de1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.IOUtils;
@@ -290,7 +291,7 @@ public class TestINodeFile {
       iNodes[i] = new INodeFile(i, null, perm, 0L, 0L, null, replication,
           preferredBlockSize);
       iNodes[i].setLocalName(DFSUtil.string2Bytes(fileNamePrefix + i));
-      BlockInfo newblock = new BlockInfo(replication);
+      BlockInfo newblock = new BlockInfoContiguous(replication);
       iNodes[i].addBlock(newblock);
     }
     

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
index 0d726e9..106edad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
@@ -148,7 +149,7 @@ public class TestTruncateQuotaUpdate {
 
   private BlockInfo newBlock(long size, short replication) {
     Block b = new Block(++nextMockBlockId, size, ++nextMockGenstamp);
-    return new BlockInfo(b, replication);
+    return new BlockInfoContiguous(b, replication);
   }
 
   private static void addSnapshotFeature(INodeFile file, BlockInfo[] blocks) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
index ac81488..0d6298f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
@@ -21,6 +21,7 @@ import com.google.common.collect.Lists;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
@@ -50,7 +51,7 @@ public class TestFileWithSnapshotFeature {
     BlockStoragePolicySuite bsps = mock(BlockStoragePolicySuite.class);
     BlockStoragePolicy bsp = mock(BlockStoragePolicy.class);
     BlockInfo[] blocks = new BlockInfo[] {
-        new BlockInfo(new Block(1, BLOCK_SIZE, 1), REPL_1)
+        new BlockInfoContiguous(new Block(1, BLOCK_SIZE, 1), REPL_1)
     };
 
     // No snapshot


[46/50] [abbrv] hadoop git commit: HADOOP-9905. remove dependency of zookeeper for hadoop-client (Contributed by Vinayakumar B)

Posted by zj...@apache.org.
HADOOP-9905. remove dependency of zookeeper for hadoop-client (Contributed by Vinayakumar B)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/850e4888
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/850e4888
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/850e4888

Branch: refs/heads/YARN-2928
Commit: 850e48887c143c28f3a1682f527ea59cf5c38560
Parents: 732a70f
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Jun 1 17:47:02 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:22 2015 -0700

----------------------------------------------------------------------
 hadoop-client/pom.xml                           | 8 ++++++++
 hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
 2 files changed, 10 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/850e4888/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml
index f8b6d97..2324dba 100644
--- a/hadoop-client/pom.xml
+++ b/hadoop-client/pom.xml
@@ -95,6 +95,10 @@
           <groupId>com.jcraft</groupId>
           <artifactId>jsch</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.zookeeper</groupId>
+          <artifactId>zookeeper</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -171,6 +175,10 @@
           <groupId>io.netty</groupId>
           <artifactId>netty</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.zookeeper</groupId>
+          <artifactId>zookeeper</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/850e4888/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 00075b4..44c2969 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -480,6 +480,8 @@ Trunk (Unreleased)
     HADOOP-12022. fix site -Pdocs -Pdist in hadoop-project-dist; cleanout
     remaining forrest bits (aw)
 
+    HADOOP-9905. remove dependency of zookeeper for hadoop-client (vinayakumarb)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)


[23/50] [abbrv] hadoop git commit: HADOOP-11934. Use of JavaKeyStoreProvider in LdapGroupsMapping causes infinite loop. Contributed by Larry McCay.

Posted by zj...@apache.org.
HADOOP-11934. Use of JavaKeyStoreProvider in LdapGroupsMapping causes infinite loop. Contributed by Larry McCay.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3fcb33b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3fcb33b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3fcb33b3

Branch: refs/heads/YARN-2928
Commit: 3fcb33b37d05bd6af730fd6dfbc054c2686a7553
Parents: b17498d
Author: cnauroth <cn...@apache.org>
Authored: Thu May 28 15:01:42 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 .../alias/AbstractJavaKeyStoreProvider.java     | 344 +++++++++++++++++++
 .../security/alias/JavaKeyStoreProvider.java    | 257 ++------------
 .../alias/LocalJavaKeyStoreProvider.java        | 192 +++++++++++
 ...oop.security.alias.CredentialProviderFactory |   1 +
 .../alias/TestCredentialProviderFactory.java    |  25 +-
 6 files changed, 597 insertions(+), 225 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 71b71a0..af82c0b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -832,6 +832,9 @@ Release 2.7.1 - UNRELEASED
     HADOOP-11973. Ensure ZkDelegationTokenSecretManager namespace znodes get
     created with ACLs. (Gregory Chanan via asuresh)
 
+    HADOOP-11934. Use of JavaKeyStoreProvider in LdapGroupsMapping causes
+    infinite loop. (Larry McCay via cnauroth)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
new file mode 100644
index 0000000..9251044
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
+
+import com.google.common.base.Charsets;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Abstract class for implementing credential providers that are based on
+ * Java Keystores as the underlying credential store.
+ *
+ * The password for the keystore is taken from the HADOOP_CREDSTORE_PASSWORD
+ * environment variable with a default of 'none'.
+ *
+ * It is expected that for access to credential protected resource to copy the
+ * creds from the original provider into the job's Credentials object, which is
+ * accessed via the UserProvider. Therefore, these providers won't be directly
+ * used by MapReduce tasks.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
+  public static final String CREDENTIAL_PASSWORD_NAME =
+      "HADOOP_CREDSTORE_PASSWORD";
+  public static final String KEYSTORE_PASSWORD_FILE_KEY =
+      "hadoop.security.credstore.java-keystore-provider.password-file";
+  public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
+
+  private Path path;
+  private final URI uri;
+  private final KeyStore keyStore;
+  private char[] password = null;
+  private boolean changed = false;
+  private Lock readLock;
+  private Lock writeLock;
+
+  protected AbstractJavaKeyStoreProvider(URI uri, Configuration conf)
+      throws IOException {
+    this.uri = uri;
+    initFileSystem(uri, conf);
+    // Get the password from the user's environment
+    if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
+      password = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
+    }
+    // if not in ENV get check for file
+    if (password == null) {
+      String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
+      if (pwFile != null) {
+        ClassLoader cl = Thread.currentThread().getContextClassLoader();
+        URL pwdFile = cl.getResource(pwFile);
+        if (pwdFile != null) {
+          try (InputStream is = pwdFile.openStream()) {
+            password = IOUtils.toString(is).trim().toCharArray();
+          }
+        }
+      }
+    }
+    if (password == null) {
+      password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
+    }
+    try {
+      keyStore = KeyStore.getInstance("jceks");
+      if (keystoreExists()) {
+        stashOriginalFilePermissions();
+        try (InputStream in = getInputStreamForFile()) {
+          keyStore.load(in, password);
+        }
+      } else {
+        createPermissions("700");
+        // required to create an empty keystore. *sigh*
+        keyStore.load(null, password);
+      }
+    } catch (KeyStoreException e) {
+      throw new IOException("Can't create keystore", e);
+    } catch (NoSuchAlgorithmException e) {
+      throw new IOException("Can't load keystore " + getPathAsString(), e);
+    } catch (CertificateException e) {
+      throw new IOException("Can't load keystore " + getPathAsString(), e);
+    }
+    ReadWriteLock lock = new ReentrantReadWriteLock(true);
+    readLock = lock.readLock();
+    writeLock = lock.writeLock();
+  }
+
+  public Path getPath() {
+    return path;
+  }
+
+  public void setPath(Path p) {
+    this.path = p;
+  }
+
+  public char[] getPassword() {
+    return password;
+  }
+
+  public void setPassword(char[] pass) {
+    this.password = pass;
+  }
+
+  public boolean isChanged() {
+    return changed;
+  }
+
+  public void setChanged(boolean chg) {
+    this.changed = chg;
+  }
+
+  public Lock getReadLock() {
+    return readLock;
+  }
+
+  public void setReadLock(Lock rl) {
+    this.readLock = rl;
+  }
+
+  public Lock getWriteLock() {
+    return writeLock;
+  }
+
+  public void setWriteLock(Lock wl) {
+    this.writeLock = wl;
+  }
+
+  public URI getUri() {
+    return uri;
+  }
+
+  public KeyStore getKeyStore() {
+    return keyStore;
+  }
+
+  public Map<String, CredentialEntry> getCache() {
+    return cache;
+  }
+
+  private final Map<String, CredentialEntry> cache =
+      new HashMap<String, CredentialEntry>();
+
+  protected final String getPathAsString() {
+    return getPath().toString();
+  }
+
+  protected abstract String getSchemeName();
+
+  protected abstract OutputStream getOutputStreamForKeystore()
+      throws IOException;
+
+  protected abstract boolean keystoreExists() throws IOException;
+
+  protected abstract InputStream getInputStreamForFile() throws IOException;
+
+  protected abstract void createPermissions(String perms) throws IOException;
+
+  protected abstract void stashOriginalFilePermissions() throws IOException;
+
+  protected void initFileSystem(URI keystoreUri, Configuration conf)
+      throws IOException {
+    path = ProviderUtils.unnestUri(keystoreUri);
+  }
+
+  @Override
+  public CredentialEntry getCredentialEntry(String alias)
+      throws IOException {
+    readLock.lock();
+    try {
+      SecretKeySpec key = null;
+      try {
+        if (cache.containsKey(alias)) {
+          return cache.get(alias);
+        }
+        if (!keyStore.containsAlias(alias)) {
+          return null;
+        }
+        key = (SecretKeySpec) keyStore.getKey(alias, password);
+      } catch (KeyStoreException e) {
+        throw new IOException("Can't get credential " + alias + " from "
+            + getPathAsString(), e);
+      } catch (NoSuchAlgorithmException e) {
+        throw new IOException("Can't get algorithm for credential " + alias
+            + " from " + getPathAsString(), e);
+      } catch (UnrecoverableKeyException e) {
+        throw new IOException("Can't recover credential " + alias + " from "
+            + getPathAsString(), e);
+      }
+      return new CredentialEntry(alias, bytesToChars(key.getEncoded()));
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public static char[] bytesToChars(byte[] bytes) throws IOException {
+    String pass;
+    pass = new String(bytes, Charsets.UTF_8);
+    return pass.toCharArray();
+  }
+
+  @Override
+  public List<String> getAliases() throws IOException {
+    readLock.lock();
+    try {
+      ArrayList<String> list = new ArrayList<String>();
+      String alias = null;
+      try {
+        Enumeration<String> e = keyStore.aliases();
+        while (e.hasMoreElements()) {
+          alias = e.nextElement();
+          list.add(alias);
+        }
+      } catch (KeyStoreException e) {
+        throw new IOException("Can't get alias " + alias + " from "
+            + getPathAsString(), e);
+      }
+      return list;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @Override
+  public CredentialEntry createCredentialEntry(String alias, char[] credential)
+      throws IOException {
+    writeLock.lock();
+    try {
+      if (keyStore.containsAlias(alias) || cache.containsKey(alias)) {
+        throw new IOException("Credential " + alias + " already exists in "
+            + this);
+      }
+      return innerSetCredential(alias, credential);
+    } catch (KeyStoreException e) {
+      throw new IOException("Problem looking up credential " + alias + " in "
+          + this, e);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public void deleteCredentialEntry(String name) throws IOException {
+    writeLock.lock();
+    try {
+      try {
+        if (keyStore.containsAlias(name)) {
+          keyStore.deleteEntry(name);
+        } else {
+          throw new IOException("Credential " + name + " does not exist in "
+              + this);
+        }
+      } catch (KeyStoreException e) {
+        throw new IOException("Problem removing " + name + " from " + this, e);
+      }
+      cache.remove(name);
+      changed = true;
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  CredentialEntry innerSetCredential(String alias, char[] material)
+      throws IOException {
+    writeLock.lock();
+    try {
+      keyStore.setKeyEntry(alias,
+          new SecretKeySpec(new String(material).getBytes("UTF-8"), "AES"),
+          password, null);
+    } catch (KeyStoreException e) {
+      throw new IOException("Can't store credential " + alias + " in " + this,
+          e);
+    } finally {
+      writeLock.unlock();
+    }
+    changed = true;
+    return new CredentialEntry(alias, material);
+  }
+
+  @Override
+  public void flush() throws IOException {
+    writeLock.lock();
+    try {
+      if (!changed) {
+        return;
+      }
+      // write out the keystore
+      try (OutputStream out = getOutputStreamForKeystore()) {
+        keyStore.store(out, password);
+      } catch (KeyStoreException e) {
+        throw new IOException("Can't store keystore " + this, e);
+      } catch (NoSuchAlgorithmException e) {
+        throw new IOException("No such algorithm storing keystore " + this, e);
+      } catch (CertificateException e) {
+        throw new IOException("Certificate exception storing keystore " + this,
+            e);
+      }
+      changed = false;
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return uri.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
index 5e5cebb..d02ebde 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
@@ -18,266 +18,75 @@
 
 package org.apache.hadoop.security.alias;
 
-import org.apache.commons.io.Charsets;
-import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.ProviderUtils;
 
-import javax.crypto.spec.SecretKeySpec;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.URI;
-import java.net.URL;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.UnrecoverableKeyException;
-import java.security.cert.CertificateException;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /**
- * CredentialProvider based on Java's KeyStore file format. The file may be 
+ * CredentialProvider based on Java's KeyStore file format. The file may be
  * stored in any Hadoop FileSystem using the following name mangling:
- *  jceks://hdfs@nn1.example.com/my/creds.jceks -> hdfs://nn1.example.com/my/creds.jceks
- *  jceks://file/home/larry/creds.jceks -> file:///home/larry/creds.jceks
- *
- * The password for the keystore is taken from the HADOOP_CREDSTORE_PASSWORD
- * environment variable with a default of 'none'.
- *
- * It is expected that for access to credential protected resource to copy the 
- * creds from the original provider into the job's Credentials object, which is
- * accessed via the UserProvider. Therefore, this provider won't be directly 
- * used by MapReduce tasks.
+ * jceks://hdfs@nn1.example.com/my/creds.jceks ->
+ * hdfs://nn1.example.com/my/creds.jceks jceks://file/home/larry/creds.jceks ->
+ * file:///home/larry/creds.jceks
  */
 @InterfaceAudience.Private
-public class JavaKeyStoreProvider extends CredentialProvider {
+public class JavaKeyStoreProvider extends AbstractJavaKeyStoreProvider {
   public static final String SCHEME_NAME = "jceks";
-  public static final String CREDENTIAL_PASSWORD_NAME =
-      "HADOOP_CREDSTORE_PASSWORD";
-  public static final String KEYSTORE_PASSWORD_FILE_KEY =
-      "hadoop.security.credstore.java-keystore-provider.password-file";
-  public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
-
-  private final URI uri;
-  private final Path path;
-  private final FileSystem fs;
-  private final FsPermission permissions;
-  private final KeyStore keyStore;
-  private char[] password = null;
-  private boolean changed = false;
-  private Lock readLock;
-  private Lock writeLock;
-
-  private final Map<String, CredentialEntry> cache = new HashMap<String, CredentialEntry>();
-
-  private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
-    this.uri = uri;
-    path = ProviderUtils.unnestUri(uri);
-    fs = path.getFileSystem(conf);
-    // Get the password from the user's environment
-    if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
-      password = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
-    }
-    // if not in ENV get check for file
-    if (password == null) {
-      String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
-      if (pwFile != null) {
-        ClassLoader cl = Thread.currentThread().getContextClassLoader();
-        URL pwdFile = cl.getResource(pwFile);
-        if (pwdFile != null) {
-          try (InputStream is = pwdFile.openStream()) {
-            password = IOUtils.toString(is).trim().toCharArray();
-          }
-        }
-      }
-    }
-    if (password == null) {
-      password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
-    }
 
-    try {
-      keyStore = KeyStore.getInstance(SCHEME_NAME);
-      if (fs.exists(path)) {
-        // save off permissions in case we need to
-        // rewrite the keystore in flush()
-        FileStatus s = fs.getFileStatus(path);
-        permissions = s.getPermission();
+  private FileSystem fs;
+  private FsPermission permissions;
 
-        try (FSDataInputStream in = fs.open(path)) {
-          keyStore.load(in, password);
-        }
-      } else {
-        permissions = new FsPermission("700");
-        // required to create an empty keystore. *sigh*
-        keyStore.load(null, password);
-      }
-    } catch (KeyStoreException e) {
-      throw new IOException("Can't create keystore", e);
-    } catch (NoSuchAlgorithmException e) {
-      throw new IOException("Can't load keystore " + path, e);
-    } catch (CertificateException e) {
-      throw new IOException("Can't load keystore " + path, e);
-    }
-    ReadWriteLock lock = new ReentrantReadWriteLock(true);
-    readLock = lock.readLock();
-    writeLock = lock.writeLock();
+  private JavaKeyStoreProvider(URI uri, Configuration conf)
+      throws IOException {
+    super(uri, conf);
   }
 
   @Override
-  public CredentialEntry getCredentialEntry(String alias) throws IOException {
-    readLock.lock();
-    try {
-      SecretKeySpec key = null;
-      try {
-        if (cache.containsKey(alias)) {
-          return cache.get(alias);
-        }
-        if (!keyStore.containsAlias(alias)) {
-          return null;
-        }
-        key = (SecretKeySpec) keyStore.getKey(alias, password);
-      } catch (KeyStoreException e) {
-        throw new IOException("Can't get credential " + alias + " from " +
-                              path, e);
-      } catch (NoSuchAlgorithmException e) {
-        throw new IOException("Can't get algorithm for credential " + alias + " from " +
-                              path, e);
-      } catch (UnrecoverableKeyException e) {
-        throw new IOException("Can't recover credential " + alias + " from " + path, e);
-      }
-      return new CredentialEntry(alias, bytesToChars(key.getEncoded()));
-    } 
-    finally {
-      readLock.unlock();
-    }
-  }
-  
-  public static char[] bytesToChars(byte[] bytes) {
-    String pass = new String(bytes, Charsets.UTF_8);
-    return pass.toCharArray();
+  protected String getSchemeName() {
+    return SCHEME_NAME;
   }
 
   @Override
-  public List<String> getAliases() throws IOException {
-    readLock.lock();
-    try {
-      ArrayList<String> list = new ArrayList<String>();
-      String alias = null;
-      try {
-        Enumeration<String> e = keyStore.aliases();
-        while (e.hasMoreElements()) {
-           alias = e.nextElement();
-           list.add(alias);
-        }
-      } catch (KeyStoreException e) {
-        throw new IOException("Can't get alias " + alias + " from " + path, e);
-      }
-      return list;
-    }
-    finally {
-      readLock.unlock();
-    }
+  protected OutputStream getOutputStreamForKeystore() throws IOException {
+    FSDataOutputStream out = FileSystem.create(fs, getPath(), permissions);
+    return out;
   }
 
   @Override
-  public CredentialEntry createCredentialEntry(String alias, char[] credential)
-      throws IOException {
-    writeLock.lock();
-    try {
-      if (keyStore.containsAlias(alias) || cache.containsKey(alias)) {
-        throw new IOException("Credential " + alias + " already exists in " + this);
-      }
-      return innerSetCredential(alias, credential);
-    } catch (KeyStoreException e) {
-      throw new IOException("Problem looking up credential " + alias + " in " + this,
-          e);
-    } finally {
-      writeLock.unlock();
-    }
+  protected boolean keystoreExists() throws IOException {
+    return fs.exists(getPath());
   }
 
   @Override
-  public void deleteCredentialEntry(String name) throws IOException {
-    writeLock.lock();
-    try {
-      try {
-        if (keyStore.containsAlias(name)) {
-          keyStore.deleteEntry(name);
-        }
-        else {
-          throw new IOException("Credential " + name + " does not exist in " + this);
-        }
-      } catch (KeyStoreException e) {
-        throw new IOException("Problem removing " + name + " from " +
-            this, e);
-      }
-      cache.remove(name);
-      changed = true;
-    }
-    finally {
-      writeLock.unlock();
-    }
+  protected InputStream getInputStreamForFile() throws IOException {
+    return fs.open(getPath());
   }
 
-  CredentialEntry innerSetCredential(String alias, char[] material)
-      throws IOException {
-    writeLock.lock();
-    try {
-      keyStore.setKeyEntry(alias, new SecretKeySpec(
-          new String(material).getBytes("UTF-8"), "AES"),
-          password, null);
-    } catch (KeyStoreException e) {
-      throw new IOException("Can't store credential " + alias + " in " + this,
-          e);
-    } finally {
-      writeLock.unlock();
-    }
-    changed = true;
-    return new CredentialEntry(alias, material);
+  @Override
+  protected void createPermissions(String perms) {
+    permissions = new FsPermission(perms);
   }
 
   @Override
-  public void flush() throws IOException {
-    writeLock.lock();
-    try {
-      if (!changed) {
-        return;
-      }
-      // write out the keystore
-      try (FSDataOutputStream out = FileSystem.create(fs, path, permissions)) {
-        keyStore.store(out, password);
-      } catch (KeyStoreException e) {
-        throw new IOException("Can't store keystore " + this, e);
-      } catch (NoSuchAlgorithmException e) {
-        throw new IOException("No such algorithm storing keystore " + this, e);
-      } catch (CertificateException e) {
-        throw new IOException("Certificate exception storing keystore " + this,
-            e);
-      }
-      changed = false;
-    }
-    finally {
-      writeLock.unlock();
-    }
+  protected void stashOriginalFilePermissions() throws IOException {
+    // save off permissions in case we need to
+    // rewrite the keystore in flush()
+    FileStatus s = fs.getFileStatus(getPath());
+    permissions = s.getPermission();
   }
 
-  @Override
-  public String toString() {
-    return uri.toString();
+  protected void initFileSystem(URI uri, Configuration conf)
+      throws IOException {
+    super.initFileSystem(uri, conf);
+    fs = getPath().getFileSystem(conf);
   }
 
   /**
@@ -286,7 +95,7 @@ public class JavaKeyStoreProvider extends CredentialProvider {
   public static class Factory extends CredentialProviderFactory {
     @Override
     public CredentialProvider createProvider(URI providerName,
-                                      Configuration conf) throws IOException {
+        Configuration conf) throws IOException {
       if (SCHEME_NAME.equals(providerName.getScheme())) {
         return new JavaKeyStoreProvider(providerName, conf);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
new file mode 100644
index 0000000..3840979
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.EnumSet;
+
+/**
+ * CredentialProvider based on Java's KeyStore file format. The file may be
+ * stored only on the local filesystem using the following name mangling:
+ * localjceks://file/home/larry/creds.jceks -> file:///home/larry/creds.jceks
+ */
+@InterfaceAudience.Private
+public final class LocalJavaKeyStoreProvider extends
+    AbstractJavaKeyStoreProvider {
+  public static final String SCHEME_NAME = "localjceks";
+  private File file;
+  private Set<PosixFilePermission> permissions;
+
+  private LocalJavaKeyStoreProvider(URI uri, Configuration conf)
+      throws IOException {
+    super(uri, conf);
+  }
+
+  @Override
+  protected String getSchemeName() {
+    return SCHEME_NAME;
+  }
+
+  @Override
+  protected OutputStream getOutputStreamForKeystore() throws IOException {
+    FileOutputStream out = new FileOutputStream(file);
+    return out;
+  }
+
+  @Override
+  protected boolean keystoreExists() throws IOException {
+    return file.exists();
+  }
+
+  @Override
+  protected InputStream getInputStreamForFile() throws IOException {
+    FileInputStream is = new FileInputStream(file);
+    return is;
+  }
+
+  @Override
+  protected void createPermissions(String perms) throws IOException {
+    int mode = 700;
+    try {
+      mode = Integer.parseInt(perms, 8);
+    } catch (NumberFormatException nfe) {
+      throw new IOException("Invalid permissions mode provided while "
+          + "trying to createPermissions", nfe);
+    }
+    permissions = modeToPosixFilePermission(mode);
+  }
+
+  @Override
+  protected void stashOriginalFilePermissions() throws IOException {
+    // save off permissions in case we need to
+    // rewrite the keystore in flush()
+    if (!Shell.WINDOWS) {
+      Path path = Paths.get(file.getCanonicalPath());
+      permissions = Files.getPosixFilePermissions(path);
+    } else {
+      // On Windows, the JDK does not support the POSIX file permission APIs.
+      // Instead, we can do a winutils call and translate.
+      String[] cmd = Shell.getGetPermissionCommand();
+      String[] args = new String[cmd.length + 1];
+      System.arraycopy(cmd, 0, args, 0, cmd.length);
+      args[cmd.length] = file.getCanonicalPath();
+      String out = Shell.execCommand(args);
+      StringTokenizer t = new StringTokenizer(out, Shell.TOKEN_SEPARATOR_REGEX);
+      // The winutils output consists of 10 characters because of the leading
+      // directory indicator, i.e. "drwx------".  The JDK parsing method expects
+      // a 9-character string, so remove the leading character.
+      String permString = t.nextToken().substring(1);
+      permissions = PosixFilePermissions.fromString(permString);
+    }
+  }
+
+  @Override
+  protected void initFileSystem(URI uri, Configuration conf)
+      throws IOException {
+    super.initFileSystem(uri, conf);
+    try {
+      file = new File(new URI(getPath().toString()));
+    } catch (URISyntaxException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @Override
+  public void flush() throws IOException {
+    super.flush();
+    if (!Shell.WINDOWS) {
+      Files.setPosixFilePermissions(Paths.get(file.getCanonicalPath()),
+          permissions);
+    } else {
+      // FsPermission expects a 10-character string because of the leading
+      // directory indicator, i.e. "drwx------". The JDK toString method returns
+      // a 9-character string, so prepend a leading character.
+      FsPermission fsPermission = FsPermission.valueOf(
+          "-" + PosixFilePermissions.toString(permissions));
+      FileUtil.setPermission(file, fsPermission);
+    }
+  }
+
+  /**
+   * The factory to create JksProviders, which is used by the ServiceLoader.
+   */
+  public static class Factory extends CredentialProviderFactory {
+    @Override
+    public CredentialProvider createProvider(URI providerName,
+        Configuration conf) throws IOException {
+      if (SCHEME_NAME.equals(providerName.getScheme())) {
+        return new LocalJavaKeyStoreProvider(providerName, conf);
+      }
+      return null;
+    }
+  }
+
+  private static Set<PosixFilePermission> modeToPosixFilePermission(
+      int mode) {
+    Set<PosixFilePermission> perms = EnumSet.noneOf(PosixFilePermission.class);
+    if ((mode & 0001) != 0) {
+      perms.add(PosixFilePermission.OTHERS_EXECUTE);
+    }
+    if ((mode & 0002) != 0) {
+      perms.add(PosixFilePermission.OTHERS_WRITE);
+    }
+    if ((mode & 0004) != 0) {
+      perms.add(PosixFilePermission.OTHERS_READ);
+    }
+    if ((mode & 0010) != 0) {
+      perms.add(PosixFilePermission.GROUP_EXECUTE);
+    }
+    if ((mode & 0020) != 0) {
+      perms.add(PosixFilePermission.GROUP_WRITE);
+    }
+    if ((mode & 0040) != 0) {
+      perms.add(PosixFilePermission.GROUP_READ);
+    }
+    if ((mode & 0100) != 0) {
+      perms.add(PosixFilePermission.OWNER_EXECUTE);
+    }
+    if ((mode & 0200) != 0) {
+      perms.add(PosixFilePermission.OWNER_WRITE);
+    }
+    if ((mode & 0400) != 0) {
+      perms.add(PosixFilePermission.OWNER_READ);
+    }
+    return perms;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
index bb7366e..f673cf4 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
@@ -14,4 +14,5 @@
 # limitations under the License.
 
 org.apache.hadoop.security.alias.JavaKeyStoreProvider$Factory
+org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider$Factory
 org.apache.hadoop.security.alias.UserProvider$Factory

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
index 1eb1e8b..16cb0be 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
@@ -207,7 +207,29 @@ public class TestCredentialProviderFactory {
     checkPermissionRetention(conf, ourUrl, path);
   }
 
-  public void checkPermissionRetention(Configuration conf, String ourUrl, 
+  @Test
+  public void testLocalJksProvider() throws Exception {
+    Configuration conf = new Configuration();
+    final Path jksPath = new Path(tmpDir.toString(), "test.jks");
+    final String ourUrl =
+        LocalJavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+    File file = new File(tmpDir, "test.jks");
+    file.delete();
+    conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+    checkSpecificProvider(conf, ourUrl);
+    Path path = ProviderUtils.unnestUri(new URI(ourUrl));
+    FileSystem fs = path.getFileSystem(conf);
+    FileStatus s = fs.getFileStatus(path);
+    assertTrue("Unexpected permissions: " + s.getPermission().toString(), s.getPermission().toString().equals("rwx------"));
+    assertTrue(file + " should exist", file.isFile());
+
+    // check permission retention after explicit change
+    fs.setPermission(path, new FsPermission("777"));
+    checkPermissionRetention(conf, ourUrl, path);
+  }
+
+  public void checkPermissionRetention(Configuration conf, String ourUrl,
       Path path) throws Exception {
     CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0);
     // let's add a new credential and flush and check that permissions are still set to 777
@@ -233,3 +255,4 @@ public class TestCredentialProviderFactory {
     		"keystore.", s.getPermission().toString().equals("rwxrwxrwx"));
   }
 }
+


[03/50] [abbrv] hadoop git commit: HDFS-8482. Rename BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.

Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
index f6b18e6..0d726e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
@@ -100,7 +100,7 @@ public class TestTruncateQuotaUpdate {
   @Test
   public void testTruncateWithSnapshotAndDivergence() {
     INodeFile file = createMockFile(BLOCKSIZE * 2 + BLOCKSIZE / 2, REPLICATION);
-    BlockInfoContiguous[] blocks = new BlockInfoContiguous
+    BlockInfo[] blocks = new BlockInfo
         [file.getBlocks().length];
     System.arraycopy(file.getBlocks(), 0, blocks, 0, blocks.length);
     addSnapshotFeature(file, blocks);
@@ -130,11 +130,11 @@ public class TestTruncateQuotaUpdate {
   }
 
   private INodeFile createMockFile(long size, short replication) {
-    ArrayList<BlockInfoContiguous> blocks = new ArrayList<>();
+    ArrayList<BlockInfo> blocks = new ArrayList<>();
     long createdSize = 0;
     while (createdSize < size) {
       long blockSize = Math.min(BLOCKSIZE, size - createdSize);
-      BlockInfoContiguous bi = newBlock(blockSize, replication);
+      BlockInfo bi = newBlock(blockSize, replication);
       blocks.add(bi);
       createdSize += BLOCKSIZE;
     }
@@ -142,16 +142,16 @@ public class TestTruncateQuotaUpdate {
         .createImmutable((short) 0x1ff));
     return new INodeFile(
         ++nextMockINodeId, new byte[0], perm, 0, 0,
-        blocks.toArray(new BlockInfoContiguous[blocks.size()]), replication,
+        blocks.toArray(new BlockInfo[blocks.size()]), replication,
         BLOCKSIZE);
   }
 
-  private BlockInfoContiguous newBlock(long size, short replication) {
+  private BlockInfo newBlock(long size, short replication) {
     Block b = new Block(++nextMockBlockId, size, ++nextMockGenstamp);
-    return new BlockInfoContiguous(b, replication);
+    return new BlockInfo(b, replication);
   }
 
-  private static void addSnapshotFeature(INodeFile file, BlockInfoContiguous[] blocks) {
+  private static void addSnapshotFeature(INodeFile file, BlockInfo[] blocks) {
     FileDiff diff = mock(FileDiff.class);
     when(diff.getBlocks()).thenReturn(blocks);
     FileDiffList diffList = new FileDiffList();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
index 11b19f3..a1abd08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage;
@@ -176,7 +176,7 @@ public class SnapshotTestHelper {
    * 
    * Specific information for different types of INode: 
    * {@link INodeDirectory}:childrenSize 
-   * {@link INodeFile}: fileSize, block list. Check {@link BlockInfoContiguous#toString()}
+   * {@link INodeFile}: fileSize, block list. Check {@link BlockInfo#toString()}
    * and {@link BlockInfoContiguousUnderConstruction#toString()} for detailed information.
    * {@link FileWithSnapshot}: next link
    * </pre>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
index 8b9ebea..ac81488 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
 import com.google.common.collect.Lists;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
@@ -49,8 +49,8 @@ public class TestFileWithSnapshotFeature {
     FileDiff diff = mock(FileDiff.class);
     BlockStoragePolicySuite bsps = mock(BlockStoragePolicySuite.class);
     BlockStoragePolicy bsp = mock(BlockStoragePolicy.class);
-    BlockInfoContiguous[] blocks = new BlockInfoContiguous[] {
-        new BlockInfoContiguous(new Block(1, BLOCK_SIZE, 1), REPL_1)
+    BlockInfo[] blocks = new BlockInfo[] {
+        new BlockInfo(new Block(1, BLOCK_SIZE, 1), REPL_1)
     };
 
     // No snapshot

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
index 85072d1..7bffb33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -108,14 +108,14 @@ public class TestSnapshotBlocksMap {
      final FSDirectory dir, final BlockManager blkManager) throws Exception {
     final INodeFile file = INodeFile.valueOf(dir.getINode(path), path);
     assertEquals(numBlocks, file.getBlocks().length);
-    for(BlockInfoContiguous b : file.getBlocks()) {
+    for(BlockInfo b : file.getBlocks()) {
       assertBlockCollection(blkManager, file, b);
     }
     return file;
   }
 
   static void assertBlockCollection(final BlockManager blkManager,
-      final INodeFile file, final BlockInfoContiguous b) {
+      final INodeFile file, final BlockInfo b) {
     Assert.assertSame(b, blkManager.getStoredBlock(b));
     Assert.assertSame(file, blkManager.getBlockCollection(b));
     Assert.assertSame(file, b.getBlockCollection());
@@ -146,10 +146,10 @@ public class TestSnapshotBlocksMap {
     {
       final INodeFile f2 = assertBlockCollection(file2.toString(), 3, fsdir,
           blockmanager);
-      BlockInfoContiguous[] blocks = f2.getBlocks();
+      BlockInfo[] blocks = f2.getBlocks();
       hdfs.delete(sub2, true);
       // The INode should have been removed from the blocksMap
-      for(BlockInfoContiguous b : blocks) {
+      for(BlockInfo b : blocks) {
         assertNull(blockmanager.getBlockCollection(b));
       }
     }
@@ -177,7 +177,7 @@ public class TestSnapshotBlocksMap {
     // Check the block information for file0
     final INodeFile f0 = assertBlockCollection(file0.toString(), 4, fsdir,
         blockmanager);
-    BlockInfoContiguous[] blocks0 = f0.getBlocks();
+    BlockInfo[] blocks0 = f0.getBlocks();
     
     // Also check the block information for snapshot of file0
     Path snapshotFile0 = SnapshotTestHelper.getSnapshotPath(sub1, "s0",
@@ -187,7 +187,7 @@ public class TestSnapshotBlocksMap {
     // Delete file0
     hdfs.delete(file0, true);
     // Make sure the blocks of file0 is still in blocksMap
-    for(BlockInfoContiguous b : blocks0) {
+    for(BlockInfo b : blocks0) {
       assertNotNull(blockmanager.getBlockCollection(b));
     }
     assertBlockCollection(snapshotFile0.toString(), 4, fsdir, blockmanager);
@@ -201,7 +201,7 @@ public class TestSnapshotBlocksMap {
     hdfs.deleteSnapshot(sub1, "s1");
 
     // Make sure the first block of file0 is still in blocksMap
-    for(BlockInfoContiguous b : blocks0) {
+    for(BlockInfo b : blocks0) {
       assertNotNull(blockmanager.getBlockCollection(b));
     }
     assertBlockCollection(snapshotFile0.toString(), 4, fsdir, blockmanager);
@@ -293,7 +293,7 @@ public class TestSnapshotBlocksMap {
     hdfs.append(bar);
 
     INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
-    BlockInfoContiguous[] blks = barNode.getBlocks();
+    BlockInfo[] blks = barNode.getBlocks();
     assertEquals(1, blks.length);
     assertEquals(BLOCKSIZE, blks[0].getNumBytes());
     ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
@@ -331,7 +331,7 @@ public class TestSnapshotBlocksMap {
     hdfs.append(bar);
 
     INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
-    BlockInfoContiguous[] blks = barNode.getBlocks();
+    BlockInfo[] blks = barNode.getBlocks();
     assertEquals(1, blks.length);
     ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
     cluster.getNameNodeRpc()
@@ -370,7 +370,7 @@ public class TestSnapshotBlocksMap {
     hdfs.append(bar);
 
     INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
-    BlockInfoContiguous[] blks = barNode.getBlocks();
+    BlockInfo[] blks = barNode.getBlocks();
     assertEquals(1, blks.length);
     ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
     cluster.getNameNodeRpc()
@@ -421,7 +421,7 @@ public class TestSnapshotBlocksMap {
     out.write(testData);
     out.close();
     INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
-    BlockInfoContiguous[] blks = barNode.getBlocks();
+    BlockInfo[] blks = barNode.getBlocks();
     assertEquals(1, blks.length);
     assertEquals(testData.length, blks[0].getNumBytes());
     

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index cdd655e..139a37e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -262,12 +262,12 @@ public class TestSnapshotDeletion {
     DFSTestUtil.createFile(hdfs, tempFile, BLOCKSIZE, REPLICATION, seed);
     final INodeFile temp = TestSnapshotBlocksMap.assertBlockCollection(
         tempFile.toString(), 1, fsdir, blockmanager);
-    BlockInfoContiguous[] blocks = temp.getBlocks();
+    BlockInfo[] blocks = temp.getBlocks();
     hdfs.delete(tempDir, true);
     // check dir's quota usage
     checkQuotaUsageComputation(dir, 8, BLOCKSIZE * REPLICATION * 3);
     // check blocks of tempFile
-    for (BlockInfoContiguous b : blocks) {
+    for (BlockInfo b : blocks) {
       assertNull(blockmanager.getBlockCollection(b));
     }
     
@@ -344,7 +344,7 @@ public class TestSnapshotDeletion {
     // while deletion, we add diff for subsub and metaChangeFile1, and remove
     // newFile
     checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 4);
-    for (BlockInfoContiguous b : blocks) {
+    for (BlockInfo b : blocks) {
       assertNull(blockmanager.getBlockCollection(b));
     }
     
@@ -481,7 +481,7 @@ public class TestSnapshotDeletion {
     
     final INodeFile toDeleteFileNode = TestSnapshotBlocksMap
         .assertBlockCollection(toDeleteFile.toString(), 1, fsdir, blockmanager);
-    BlockInfoContiguous[] blocks = toDeleteFileNode.getBlocks();
+    BlockInfo[] blocks = toDeleteFileNode.getBlocks();
     
     // create snapshot s0 on dir
     SnapshotTestHelper.createSnapshot(hdfs, dir, "s0");
@@ -507,7 +507,7 @@ public class TestSnapshotDeletion {
     // metaChangeDir's diff, dir's diff. diskspace: remove toDeleteFile, and 
     // metaChangeFile's replication factor decreases
     checkQuotaUsageComputation(dir, 6, 2 * BLOCKSIZE * REPLICATION - BLOCKSIZE);
-    for (BlockInfoContiguous b : blocks) {
+    for (BlockInfo b : blocks) {
       assertNull(blockmanager.getBlockCollection(b));
     }
     
@@ -801,7 +801,7 @@ public class TestSnapshotDeletion {
     FileStatus statusBeforeDeletion13 = hdfs.getFileStatus(file13_s1);
     INodeFile file14Node = TestSnapshotBlocksMap.assertBlockCollection(
         file14_s2.toString(), 1, fsdir, blockmanager);
-    BlockInfoContiguous[] blocks_14 = file14Node.getBlocks();
+    BlockInfo[] blocks_14 = file14Node.getBlocks();
     TestSnapshotBlocksMap.assertBlockCollection(file15_s2.toString(), 1, fsdir,
         blockmanager);
     
@@ -838,7 +838,7 @@ public class TestSnapshotDeletion {
         modDirStr + "file15");
     assertFalse(hdfs.exists(file14_s1));
     assertFalse(hdfs.exists(file15_s1));
-    for (BlockInfoContiguous b : blocks_14) {
+    for (BlockInfo b : blocks_14) {
       assertNull(blockmanager.getBlockCollection(b));
     }
     


[40/50] [abbrv] hadoop git commit: YARN-1462. Made RM write application tags to timeline server and exposed them to users via generic history web UI and REST API. Contributed by Xuan Gong.

Posted by zj...@apache.org.
YARN-1462. Made RM write application tags to timeline server and exposed them to users via generic history web UI and REST API. Contributed by Xuan Gong.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2101ca1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2101ca1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2101ca1

Branch: refs/heads/YARN-2928
Commit: e2101ca195ab003f53d12f58d5b7604b1dc27fc9
Parents: af69d9c
Author: Zhijie Shen <zj...@apache.org>
Authored: Sat May 30 09:35:59 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:59 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/mapred/NotRunningJob.java |  2 +-
 .../mapred/TestClientServiceDelegate.java       |  4 +--
 .../apache/hadoop/mapred/TestYARNRunner.java    |  2 +-
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../yarn/api/records/ApplicationReport.java     |  4 ++-
 .../hadoop/yarn/client/ProtocolHATestBase.java  |  2 +-
 .../yarn/client/api/impl/TestAHSClient.java     |  8 ++---
 .../yarn/client/api/impl/TestYarnClient.java    |  9 ++---
 .../hadoop/yarn/client/cli/TestYarnCLI.java     | 27 ++++++++------
 .../hadoop/yarn/api/TestApplicatonReport.java   |  3 +-
 .../ApplicationHistoryManagerImpl.java          |  2 +-
 ...pplicationHistoryManagerOnTimelineStore.java | 19 ++++++++--
 ...pplicationHistoryManagerOnTimelineStore.java |  9 +++++
 .../metrics/ApplicationMetricsConstants.java    |  1 +
 .../metrics/ApplicationCreatedEvent.java        | 10 +++++-
 .../metrics/SystemMetricsPublisher.java         |  4 ++-
 .../applicationsmanager/MockAsm.java            |  2 +-
 .../metrics/TestSystemMetricsPublisher.java     | 37 ++++++++++++++++++++
 .../resourcemanager/webapp/TestRMWebApp.java    |  2 +-
 .../src/site/markdown/TimelineServer.md         | 30 ++++++++++++++--
 20 files changed, 144 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
index 03552e4..1b26cd3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
@@ -90,7 +90,7 @@ public class NotRunningJob implements MRClientProtocol {
     return ApplicationReport.newInstance(unknownAppId, unknownAttemptId,
       "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
       "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
-      YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+      YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null);
   }
 
   NotRunningJob(ApplicationReport applicationReport, JobState jobState) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
index b85f18d..c1c03df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
@@ -516,7 +516,7 @@ public class TestClientServiceDelegate {
     return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
       "appname", "host", 124, null, YarnApplicationState.FINISHED,
       "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
-      "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+      "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null);
   }
 
   private ApplicationReport getRunningApplicationReport(String host, int port) {
@@ -526,7 +526,7 @@ public class TestClientServiceDelegate {
     return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
       "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics",
       "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
-      YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+      YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null);
   }
 
   private ResourceMgrDelegate getRMDelegate() throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 0e53ab0..380fd62 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -196,7 +196,7 @@ public class TestYARNRunner extends TestCase {
             ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp",
                 "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp",
                 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f,
-                "tmp", null));
+                "tmp", null, null));
     yarnRunner.killJob(jobId);
     verify(clientDelegate).killJob(jobId);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2c48701..6f5faa9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -584,6 +584,9 @@ Release 2.7.1 - UNRELEASED
     YARN-3723. Need to clearly document primaryFilter and otherInfo value type.
     (Zhijie Shen via xgong)
 
+    YARN-1462. Made RM write application tags to timeline server and exposed them
+    to users via generic history web UI and REST API. (Xuan Gong via zjshen)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
index e5d7254..99e4e2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
@@ -59,7 +59,8 @@ public abstract class ApplicationReport {
       YarnApplicationState state, String diagnostics, String url,
       long startTime, long finishTime, FinalApplicationStatus finalStatus,
       ApplicationResourceUsageReport appResources, String origTrackingUrl,
-      float progress, String applicationType, Token amRmToken) {
+      float progress, String applicationType, Token amRmToken,
+      Set<String> tags) {
     ApplicationReport report = Records.newRecord(ApplicationReport.class);
     report.setApplicationId(applicationId);
     report.setCurrentApplicationAttemptId(applicationAttemptId);
@@ -80,6 +81,7 @@ public abstract class ApplicationReport {
     report.setProgress(progress);
     report.setApplicationType(applicationType);
     report.setAMRMToken(amRmToken);
+    report.setApplicationTags(tags);
     return report;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index 903dd94..2260de5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -647,7 +647,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
               "fakeQueue", "fakeApplicationName", "localhost", 0, null,
               YarnApplicationState.FINISHED, "fake an application report", "",
               1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f,
-              "fakeApplicationType", null);
+              "fakeApplicationType", null, null);
       return report;
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
index c3e3c41..12d0602 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
@@ -334,7 +334,7 @@ public class TestAHSClient {
             "queue", "appname", "host", 124, null,
             YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
             FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
-            null);
+            null, null);
       List<ApplicationReport> applicationReports =
           new ArrayList<ApplicationReport>();
       applicationReports.add(newApplicationReport);
@@ -391,7 +391,7 @@ public class TestAHSClient {
             "queue2", "appname2", "host2", 125, null,
             YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
             FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f,
-            "NON-YARN", null);
+            "NON-YARN", null, null);
       applicationReports.add(newApplicationReport2);
 
       ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -401,7 +401,7 @@ public class TestAHSClient {
             "queue3", "appname3", "host3", 126, null,
             YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
             FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f,
-            "MAPREDUCE", null);
+            "MAPREDUCE", null, null);
       applicationReports.add(newApplicationReport3);
 
       ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -411,7 +411,7 @@ public class TestAHSClient {
             "queue4", "appname4", "host4", 127, null,
             YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
             FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
-            "NON-MAPREDUCE", null);
+            "NON-MAPREDUCE", null, null);
       applicationReports.add(newApplicationReport4);
       reports = applicationReports;
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index bc40b9a..b7608b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -613,7 +613,8 @@ public class TestYarnClient {
           applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
           "user", "queue", "appname", "host", 124, null,
           YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
-          FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+          FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+          null);
       List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
       applicationReports.add(newApplicationReport);
       List<ApplicationAttemptReport> appAttempts = new ArrayList<ApplicationAttemptReport>();
@@ -689,7 +690,7 @@ public class TestYarnClient {
           "user2", "queue2", "appname2", "host2", 125, null,
           YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
           FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", 
-        null);
+          null, null);
       applicationReports.add(newApplicationReport2);
 
       ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -698,7 +699,7 @@ public class TestYarnClient {
           "user3", "queue3", "appname3", "host3", 126, null,
           YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
           FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
-        null);
+          null, null);
       applicationReports.add(newApplicationReport3);
 
       ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -709,7 +710,7 @@ public class TestYarnClient {
               "user4", "queue4", "appname4", "host4", 127, null,
               YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
               FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
-              "NON-MAPREDUCE", null);
+              "NON-MAPREDUCE", null, null);
       applicationReports.add(newApplicationReport4);
       return applicationReports;
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 1013958..70c7d82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -104,7 +104,7 @@ public class TestYarnCLI {
           "user", "queue", "appname", "host", 124, null,
           YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
           FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
-          null);
+          null, null);
       newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
       when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
           newApplicationReport);
@@ -350,7 +350,8 @@ public class TestYarnCLI {
         applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
         "user", "queue", "appname", "host", 124, null,
         YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
-        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+        null);
     List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
     applicationReports.add(newApplicationReport);
 
@@ -360,7 +361,7 @@ public class TestYarnCLI {
         "user2", "queue2", "appname2", "host2", 125, null,
         YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
         FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", 
-      null);
+        null, null);
     applicationReports.add(newApplicationReport2);
 
     ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -369,7 +370,7 @@ public class TestYarnCLI {
         "user3", "queue3", "appname3", "host3", 126, null,
         YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
         FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", 
-        null);
+        null, null);
     applicationReports.add(newApplicationReport3);
 
     ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -378,7 +379,7 @@ public class TestYarnCLI {
         "user4", "queue4", "appname4", "host4", 127, null,
         YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
         FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
-        null);
+        null, null);
     applicationReports.add(newApplicationReport4);
 
     ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
@@ -387,7 +388,7 @@ public class TestYarnCLI {
         "user5", "queue5", "appname5", "host5", 128, null,
         YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5,
         FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE",
-        null);
+        null, null);
     applicationReports.add(newApplicationReport5);
 
     ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
@@ -396,7 +397,7 @@ public class TestYarnCLI {
         "user6", "queue6", "appname6", "host6", 129, null,
         YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6,
         FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG",
-        null);
+        null, null);
     applicationReports.add(newApplicationReport6);
 
     // Test command yarn application -list
@@ -820,7 +821,8 @@ public class TestYarnCLI {
         applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
         "user", "queue", "appname", "host", 124, null,
         YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
-        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+        null);
     when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
         newApplicationReport2);
     int result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@@ -833,7 +835,8 @@ public class TestYarnCLI {
         applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
         "user", "queue", "appname", "host", 124, null,
         YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
-        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+        null);
     when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
         newApplicationReport);
     result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@@ -868,7 +871,8 @@ public class TestYarnCLI {
         applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
         "user", "queue", "appname", "host", 124, null,
         YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
-        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+        null);
     when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
         newApplicationReport2);
     int result = cli.run(new String[] { "application", "-movetoqueue",
@@ -883,7 +887,8 @@ public class TestYarnCLI {
         applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
         "user", "queue", "appname", "host", 124, null,
         YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
-        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+        null);
     when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
         newApplicationReport);
     result = cli.run(new String[] { "application", "-movetoqueue",

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
index 9302d4b..a2ea0a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
@@ -58,7 +58,8 @@ public class TestApplicatonReport {
         ApplicationReport.newInstance(appId, appAttemptId, "user", "queue",
           "appname", "host", 124, null, YarnApplicationState.FINISHED,
           "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
-          "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+          "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
+          null);
     return appReport;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index 479858f..766e0cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -139,7 +139,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
       appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
       trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(),
       appHistory.getFinalApplicationStatus(), null, "", 100,
-      appHistory.getApplicationType(), null);
+      appHistory.getApplicationType(), null, null);
   }
 
   private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 9bfd2d6..0c7fdc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -19,11 +19,14 @@
 package org.apache.hadoop.yarn.server.applicationhistoryservice;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -249,6 +252,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
     YarnApplicationState state = YarnApplicationState.ACCEPTED;
     ApplicationResourceUsageReport appResources = null;
+    Set<String> appTags = null;
     Map<ApplicationAccessType, String> appViewACLs =
         new HashMap<ApplicationAccessType, String>();
     Map<String, Object> entityInfo = entity.getOtherInfo();
@@ -270,7 +274,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
             ConverterUtils.toApplicationId(entity.getEntityId()),
             latestApplicationAttemptId, user, queue, name, null, -1, null, state,
             diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
-            null, progress, type, null), appViewACLs);
+            null, progress, type, null, appTags), appViewACLs);
       }
       if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
         queue =
@@ -295,6 +299,17 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
         appResources=ApplicationResourceUsageReport
             .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds);
       }
+      if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
+        appTags = new HashSet<String>();
+        Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
+        if (obj != null && obj instanceof Collection<?>) {
+          for(Object o : (Collection<?>)obj) {
+            if (o != null) {
+              appTags.add(o.toString());
+            }
+          }
+        }
+      }
     }
     List<TimelineEvent> events = entity.getEvents();
     if (events != null) {
@@ -347,7 +362,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
         ConverterUtils.toApplicationId(entity.getEntityId()),
         latestApplicationAttemptId, user, queue, name, null, -1, null, state,
         diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
-        null, progress, type, null), appViewACLs);
+        null, progress, type, null, appTags), appViewACLs);
   }
 
   private static ApplicationAttemptReport convertToApplicationAttemptReport(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 8672953..dbd75ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -22,7 +22,9 @@ import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
@@ -185,6 +187,9 @@ public class TestApplicationHistoryManagerOnTimelineStore {
       Assert.assertEquals(Integer.MAX_VALUE + 3L
           + +app.getApplicationId().getId(), app.getFinishTime());
       Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
+      Assert.assertEquals(2, app.getApplicationTags().size());
+      Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1"));
+      Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
       // App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
       // Nobody except admin and owner has access to the details of the app.
       if ((i ==  1 && callerUGI != null &&
@@ -471,6 +476,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
       entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
           "user2");
     }
+    Set<String> appTags = new HashSet<String>();
+    appTags.add("Test_APP_TAGS_1");
+    appTags.add("Test_APP_TAGS_2");
+    entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, appTags);
     entity.setOtherInfo(entityInfo);
     TimelineEvent tEvent = new TimelineEvent();
     tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index df8eecb..f452410 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -73,4 +73,5 @@ public class ApplicationMetricsConstants {
   public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
       "YARN_APPLICATION_LATEST_APP_ATTEMPT";
 
+  public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
index 2373b3b..7c43aa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.metrics;
 
+import java.util.Set;
+
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 
 public class ApplicationCreatedEvent extends
@@ -29,6 +31,7 @@ public class ApplicationCreatedEvent extends
   private String user;
   private String queue;
   private long submittedTime;
+  private Set<String> appTags;
 
   public ApplicationCreatedEvent(ApplicationId appId,
       String name,
@@ -36,7 +39,8 @@ public class ApplicationCreatedEvent extends
       String user,
       String queue,
       long submittedTime,
-      long createdTime) {
+      long createdTime,
+      Set<String> appTags) {
     super(SystemMetricsEventType.APP_CREATED, createdTime);
     this.appId = appId;
     this.name = name;
@@ -44,6 +48,7 @@ public class ApplicationCreatedEvent extends
     this.user = user;
     this.queue = queue;
     this.submittedTime = submittedTime;
+    this.appTags = appTags;
   }
 
   @Override
@@ -75,4 +80,7 @@ public class ApplicationCreatedEvent extends
     return submittedTime;
   }
 
+  public Set<String> getAppTags() {
+    return appTags;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 2828aec..63461b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -107,7 +107,7 @@ public class SystemMetricsPublisher extends CompositeService {
               app.getUser(),
               app.getQueue(),
               app.getSubmitTime(),
-              createdTime));
+              createdTime, app.getApplicationTags()));
     }
   }
 
@@ -252,6 +252,8 @@ public class SystemMetricsPublisher extends CompositeService {
         event.getQueue());
     entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
         event.getSubmittedTime());
+    entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
+        event.getAppTags());
     entity.setOtherInfo(entityInfo);
     TimelineEvent tEvent = new TimelineEvent();
     tEvent.setEventType(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index e52b054..34247e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -313,7 +313,7 @@ public abstract class MockAsm extends MockApps {
             getName(), null, 0, null, null, getDiagnostics().toString(), 
             getTrackingUrl(), getStartTime(), getFinishTime(), 
             getFinalApplicationStatus(), usageReport , null, getProgress(),
-            type, null);
+            type, null, null);
         return report;
       }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 7c4b5e9..b122bc4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -21,7 +21,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import java.util.Collection;
 import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -143,6 +147,8 @@ public class TestSystemMetricsPublisher {
       Assert.assertEquals(app.getSubmitTime(),
           entity.getOtherInfo().get(
               ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO));
+      Assert.assertTrue(verifyAppTags(app.getApplicationTags(),
+          entity.getOtherInfo()));
       if (i == 1) {
         Assert.assertEquals("uers1,user2",
             entity.getOtherInfo().get(
@@ -352,6 +358,10 @@ public class TestSystemMetricsPublisher {
         FinalApplicationStatus.UNDEFINED);
     when(app.getRMAppMetrics()).thenReturn(
         new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
+    Set<String> appTags = new HashSet<String>();
+    appTags.add("test");
+    appTags.add("tags");
+    when(app.getApplicationTags()).thenReturn(appTags);
     return app;
   }
 
@@ -392,4 +402,31 @@ public class TestSystemMetricsPublisher {
     return container;
   }
 
+  private static boolean verifyAppTags(Set<String> appTags,
+      Map<String, Object> entityInfo) {
+    if (!entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
+      return false;
+    }
+    Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
+    if (obj instanceof Collection<?>) {
+      Collection<?> collection = (Collection<?>) obj;
+      if (collection.size() != appTags.size()) {
+        return false;
+      }
+      for (String appTag : appTags) {
+        boolean match = false;
+        for (Object o : collection) {
+          if (o.toString().equals(appTag)) {
+            match = true;
+            break;
+          }
+        }
+        if (!match) {
+          return false;
+        }
+      }
+      return true;
+    }
+    return false;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index 0290421..38eebfe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -253,7 +253,7 @@ public class TestRMWebApp {
               app.getStartTime(), app.getFinishTime(),
               app.getFinalApplicationStatus(),
               (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
-              app.getProgress(), app.getApplicationType(), (Token) null);
+              app.getProgress(), app.getApplicationType(), (Token) null, null);
       appReports.add(appReport);
     }
     GetApplicationsResponse response = mock(GetApplicationsResponse.class);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 90f9512..1b66606 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -1125,7 +1125,27 @@ Response Body:
           "submittedTime":1430424769395,
           "startedTime":1430424769395,
           "finishedTime":1430424776594,
-          "elapsedTime":7199
+          "elapsedTime":7199},
+          {
+          "appId":"application_1430424020775_0001",
+          "currentAppAttemptId":"appattempt_1430424020775_0001_000001",
+          "user":"zshen",
+          "name":"QuasiMonteCarlo",
+          "queue":"default",
+          "type":"MAPREDUCE",
+          "host":"localhost",
+          "rpcPort":56264,
+          "appState":"FINISHED",
+          "progress":100.0,
+          "diagnosticsInfo":"",
+          "originalTrackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:19888/jobhistory/job/job_1430424020775_0001",
+          "trackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:8088/proxy/application_1430424020775_0001/",
+          "finalAppStatus":"SUCCEEDED",
+          "submittedTime":1430424053809,
+          "startedTime":1430424072153,
+          "finishedTime":1430424776594,
+          "elapsedTime":18344,
+          "applicationTags":"mrapplication,ta-example"
           }
       ]
     }
@@ -1227,6 +1247,7 @@ Response Body:
         <startedTime>1430424053809</startedTime>
         <finishedTime>1430424072153</finishedTime>
         <elapsedTime>18344</elapsedTime>
+        <applicationTags>mrapplication,ta-example</applicationTags>
       </app>
     </apps>
 
@@ -1275,7 +1296,8 @@ None
 | `allocatedVCores` | int | The sum of virtual cores allocated to the application's running containers |
 | `currentAppAttemptId` | string | The latest application attempt ID |
 | `host` | string | The host of the ApplicationMaster |
-| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared. |
+| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared |
+| `applicationTags` | string | The application tags. |
 
 ### Response Examples:
 
@@ -1311,7 +1333,8 @@ Response Body:
       "submittedTime": 1430424053809,
       "startedTime": 1430424053809,
       "finishedTime": 1430424072153,
-      "elapsedTime": 18344
+      "elapsedTime": 18344,
+      "applicationTags": mrapplication,tag-example
     }
 
 #### XML response
@@ -1349,6 +1372,7 @@ Response Body:
        <startedTime>1430424053809</startedTime>
        <finishedTime>1430424072153</finishedTime>
        <elapsedTime>18344</elapsedTime>
+       <applicationTags>mrapplication,ta-example</applicationTags>
      </app>
 
 ## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List


[33/50] [abbrv] hadoop git commit: HDFS-7609. Avoid retry cache collision when Standby NameNode loading edits. Contributed by Ming Ma.

Posted by zj...@apache.org.
HDFS-7609. Avoid retry cache collision when Standby NameNode loading edits. Contributed by Ming Ma.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58b91c94
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58b91c94
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58b91c94

Branch: refs/heads/YARN-2928
Commit: 58b91c94086f7e2cacd07087c9e7a0d28b2355d3
Parents: ab02e31
Author: Jing Zhao <ji...@apache.org>
Authored: Fri May 29 11:05:13 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 ++
 .../hdfs/server/namenode/FSNamesystem.java      | 20 -----------
 .../hdfs/server/namenode/NameNodeRpcServer.java | 22 ++++++++++--
 .../namenode/ha/TestRetryCacheWithHA.java       | 37 ++++++++++++++++++--
 4 files changed, 58 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b91c94/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 23699a3..cc8235c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -822,6 +822,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-7401. Add block info to DFSInputStream' WARN message when it adds
     node to deadNodes (Arshad Mohammad via vinayakumarb)
 
+    HDFS-7609. Avoid retry cache collision when Standby NameNode loading edits.
+    (Ming Ma via jing9)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b91c94/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 798f8d5..5ed069d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1887,7 +1887,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
    */
   void concat(String target, String [] srcs, boolean logRetryCache)
       throws IOException {
-    checkOperation(OperationCategory.WRITE);
     waitForLoadingFSImage();
     HdfsFileStatus stat = null;
     boolean success = false;
@@ -2162,7 +2161,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     }
     waitForLoadingFSImage();
     HdfsFileStatus auditStat = null;
-    checkOperation(OperationCategory.WRITE);
     writeLock();
     try {
       checkOperation(OperationCategory.WRITE);
@@ -2379,7 +2377,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
       throw new InvalidPathException(src);
     }
     blockManager.verifyReplication(src, replication, clientMachine);
-    checkOperation(OperationCategory.WRITE);
     if (blockSize < minBlockSize) {
       throw new IOException("Specified block size is less than configured" +
           " minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY
@@ -2786,7 +2783,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     LocatedBlock lb = null;
     HdfsFileStatus stat = null;
     FSPermissionChecker pc = getPermissionChecker();
-    checkOperation(OperationCategory.WRITE);
     byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
     writeLock();
     try {
@@ -3081,7 +3077,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   boolean renameTo(String src, String dst, boolean logRetryCache)
       throws IOException {
     waitForLoadingFSImage();
-    checkOperation(OperationCategory.WRITE);
     FSDirRenameOp.RenameOldResult ret = null;
     writeLock();
     try {
@@ -3107,7 +3102,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
                 boolean logRetryCache, Options.Rename... options)
       throws IOException {
     waitForLoadingFSImage();
-    checkOperation(OperationCategory.WRITE);
     Map.Entry<BlocksMapUpdateInfo, HdfsFileStatus> res = null;
     writeLock();
     try {
@@ -3144,7 +3138,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   boolean delete(String src, boolean recursive, boolean logRetryCache)
       throws IOException {
     waitForLoadingFSImage();
-    checkOperation(OperationCategory.WRITE);
     BlocksMapUpdateInfo toRemovedBlocks = null;
     writeLock();
     boolean ret = false;
@@ -5762,8 +5755,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
       String clientName, ExtendedBlock oldBlock, ExtendedBlock newBlock,
       DatanodeID[] newNodes, String[] newStorageIDs, boolean logRetryCache)
       throws IOException {
-    checkOperation(OperationCategory.WRITE);
-
     LOG.info("updatePipeline(" + oldBlock.getLocalBlock()
              + ", newGS=" + newBlock.getGenerationStamp()
              + ", newLength=" + newBlock.getNumBytes()
@@ -6744,7 +6735,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   void renameSnapshot(
       String path, String snapshotOldName, String snapshotNewName,
       boolean logRetryCache) throws IOException {
-    checkOperation(OperationCategory.WRITE);
     boolean success = false;
     writeLock();
     try {
@@ -6832,7 +6822,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
    */
   void deleteSnapshot(String snapshotRoot, String snapshotName,
       boolean logRetryCache) throws IOException {
-    checkOperation(OperationCategory.WRITE);
     boolean success = false;
     writeLock();
     BlocksMapUpdateInfo blocksToBeDeleted = null;
@@ -7057,7 +7046,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   long addCacheDirective(CacheDirectiveInfo directive,
                          EnumSet<CacheFlag> flags, boolean logRetryCache)
       throws IOException {
-    checkOperation(OperationCategory.WRITE);
     CacheDirectiveInfo effectiveDirective = null;
     if (!flags.contains(CacheFlag.FORCE)) {
       cacheManager.waitForRescanIfNeeded();
@@ -7085,7 +7073,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
 
   void modifyCacheDirective(CacheDirectiveInfo directive,
       EnumSet<CacheFlag> flags, boolean logRetryCache) throws IOException {
-    checkOperation(OperationCategory.WRITE);
     boolean success = false;
     if (!flags.contains(CacheFlag.FORCE)) {
       cacheManager.waitForRescanIfNeeded();
@@ -7109,7 +7096,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   }
 
   void removeCacheDirective(long id, boolean logRetryCache) throws IOException {
-    checkOperation(OperationCategory.WRITE);
     boolean success = false;
     writeLock();
     try {
@@ -7148,7 +7134,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
 
   void addCachePool(CachePoolInfo req, boolean logRetryCache)
       throws IOException {
-    checkOperation(OperationCategory.WRITE);
     writeLock();
     boolean success = false;
     String poolInfoStr = null;
@@ -7170,7 +7155,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
 
   void modifyCachePool(CachePoolInfo req, boolean logRetryCache)
       throws IOException {
-    checkOperation(OperationCategory.WRITE);
     writeLock();
     boolean success = false;
     try {
@@ -7192,7 +7176,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
 
   void removeCachePool(String cachePoolName, boolean logRetryCache)
       throws IOException {
-    checkOperation(OperationCategory.WRITE);
     writeLock();
     boolean success = false;
     try {
@@ -7387,7 +7370,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     String src = srcArg;
     HdfsFileStatus resultingStat = null;
     checkSuperuserPrivilege();
-    checkOperation(OperationCategory.WRITE);
     final byte[][] pathComponents =
       FSDirectory.getPathComponentsForReservedPath(src);
     FSPermissionChecker pc = getPermissionChecker();
@@ -7473,7 +7455,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
   void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
                 boolean logRetryCache)
       throws IOException {
-    checkOperation(OperationCategory.WRITE);
     HdfsFileStatus auditStat = null;
     writeLock();
     try {
@@ -7521,7 +7502,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
 
   void removeXAttr(String src, XAttr xAttr, boolean logRetryCache)
       throws IOException {
-    checkOperation(OperationCategory.WRITE);
     HdfsFileStatus auditStat = null;
     writeLock();
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b91c94/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 0d416a6..dafa23e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -616,7 +616,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
       throw new IOException("create: Pathname too long.  Limit "
           + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
     }
-
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache, null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return (HdfsFileStatus) cacheEntry.getPayload();
@@ -647,6 +647,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
       stateChangeLog.debug("*DIR* NameNode.append: file "
           +src+" for "+clientName+" at "+clientMachine);
     }
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
         null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -794,6 +795,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
       ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs)
       throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -838,7 +840,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
       throw new IOException("rename: Pathname too long.  Limit "
           + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
     }
-
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return true; // Return previous response
@@ -859,6 +861,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   @Override // ClientProtocol
   public void concat(String trg, String[] src) throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -884,6 +887,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
       throw new IOException("rename: Pathname too long.  Limit "
           + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
     }
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -922,6 +926,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
       stateChangeLog.debug("*DIR* Namenode.delete: src=" + src
           + ", recursive=" + recursive);
     }
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return true; // Return previous response
@@ -1207,6 +1212,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   public void createSymlink(String target, String link, FsPermission dirPerms,
       boolean createParent) throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1537,6 +1543,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
       throw new IOException("createSnapshot: Pathname too long.  Limit "
           + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
     }
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
       null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1558,6 +1565,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   public void deleteSnapshot(String snapshotRoot, String snapshotName)
       throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     metrics.incrDeleteSnapshotOps();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1596,6 +1604,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
     if (snapshotNewName == null || snapshotNewName.isEmpty()) {
       throw new IOException("The new snapshot name is null or empty.");
     }
+    namesystem.checkOperation(OperationCategory.WRITE);
     metrics.incrRenameSnapshotOps();
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1635,6 +1644,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   public long addCacheDirective(
       CacheDirectiveInfo path, EnumSet<CacheFlag> flags) throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion
       (retryCache, null);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1656,6 +1666,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   public void modifyCacheDirective(
       CacheDirectiveInfo directive, EnumSet<CacheFlag> flags) throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -1673,6 +1684,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   @Override // ClientProtocol
   public void removeCacheDirective(long id) throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -1699,6 +1711,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   @Override //ClientProtocol
   public void addCachePool(CachePoolInfo info) throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1715,6 +1728,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   @Override // ClientProtocol
   public void modifyCachePool(CachePoolInfo info) throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1731,6 +1745,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   @Override // ClientProtocol
   public void removeCachePool(String cachePoolName) throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -1793,6 +1808,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   public void createEncryptionZone(String src, String keyName)
     throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return;
@@ -1824,6 +1840,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
       throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response
@@ -1853,6 +1870,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
   @Override // ClientProtocol
   public void removeXAttr(String src, XAttr xAttr) throws IOException {
     checkNNStartup();
+    namesystem.checkOperation(OperationCategory.WRITE);
     CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
     if (cacheEntry != null && cacheEntry.isSuccess()) {
       return; // Return previous response

http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b91c94/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
index e3572ab..d202fb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
@@ -215,7 +215,8 @@ public class TestRetryCacheWithHA {
   abstract class AtMostOnceOp {
     private final String name;
     final DFSClient client;
-    
+    int expectedUpdateCount = 0;
+
     AtMostOnceOp(String name, DFSClient client) {
       this.name = name;
       this.client = client;
@@ -225,6 +226,9 @@ public class TestRetryCacheWithHA {
     abstract void invoke() throws Exception;
     abstract boolean checkNamenodeBeforeReturn() throws Exception;
     abstract Object getResult();
+    int getExpectedCacheUpdateCount() {
+      return expectedUpdateCount;
+    }
   }
   
   /** createSnapshot operaiton */
@@ -604,7 +608,7 @@ public class TestRetryCacheWithHA {
   class DeleteOp extends AtMostOnceOp {
     private final String target;
     private boolean deleted;
-    
+
     DeleteOp(DFSClient client, String target) {
       super("delete", client);
       this.target = target;
@@ -614,12 +618,14 @@ public class TestRetryCacheWithHA {
     void prepare() throws Exception {
       Path p = new Path(target);
       if (!dfs.exists(p)) {
+        expectedUpdateCount++;
         DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
       }
     }
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       deleted = client.delete(target, true);
     }
 
@@ -655,12 +661,14 @@ public class TestRetryCacheWithHA {
     void prepare() throws Exception {
       Path p = new Path(target);
       if (!dfs.exists(p)) {
+        expectedUpdateCount++;
         DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
       }
     }
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       client.createSymlink(target, link, false);
     }
 
@@ -773,11 +781,13 @@ public class TestRetryCacheWithHA {
 
     @Override
     void prepare() throws Exception {
+      expectedUpdateCount++;
       dfs.addCachePool(new CachePoolInfo(directive.getPool()));
     }
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       result = client.addCacheDirective(directive, EnumSet.of(CacheFlag.FORCE));
     }
 
@@ -819,12 +829,15 @@ public class TestRetryCacheWithHA {
 
     @Override
     void prepare() throws Exception {
+      expectedUpdateCount++;
       dfs.addCachePool(new CachePoolInfo(directive.getPool()));
+      expectedUpdateCount++;
       id = client.addCacheDirective(directive, EnumSet.of(CacheFlag.FORCE));
     }
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       client.modifyCacheDirective(
           new CacheDirectiveInfo.Builder().
               setId(id).
@@ -875,12 +888,15 @@ public class TestRetryCacheWithHA {
 
     @Override
     void prepare() throws Exception {
+      expectedUpdateCount++;
       dfs.addCachePool(new CachePoolInfo(directive.getPool()));
+      expectedUpdateCount++;
       id = dfs.addCacheDirective(directive, EnumSet.of(CacheFlag.FORCE));
     }
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       client.removeCacheDirective(id);
     }
 
@@ -922,6 +938,7 @@ public class TestRetryCacheWithHA {
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       client.addCachePool(new CachePoolInfo(pool));
     }
 
@@ -954,11 +971,13 @@ public class TestRetryCacheWithHA {
 
     @Override
     void prepare() throws Exception {
+      expectedUpdateCount++;
       client.addCachePool(new CachePoolInfo(pool).setLimit(10l));
     }
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       client.modifyCachePool(new CachePoolInfo(pool).setLimit(99l));
     }
 
@@ -991,11 +1010,13 @@ public class TestRetryCacheWithHA {
 
     @Override
     void prepare() throws Exception {
+      expectedUpdateCount++;
       client.addCachePool(new CachePoolInfo(pool));
     }
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       client.removeCachePool(pool);
     }
 
@@ -1030,12 +1051,14 @@ public class TestRetryCacheWithHA {
     void prepare() throws Exception {
       Path p = new Path(src);
       if (!dfs.exists(p)) {
+        expectedUpdateCount++;
         DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
       }
     }
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       client.setXAttr(src, "user.key", "value".getBytes(),
           EnumSet.of(XAttrSetFlag.CREATE));
     }
@@ -1072,7 +1095,9 @@ public class TestRetryCacheWithHA {
     void prepare() throws Exception {
       Path p = new Path(src);
       if (!dfs.exists(p)) {
+        expectedUpdateCount++;
         DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
+        expectedUpdateCount++;
         client.setXAttr(src, "user.key", "value".getBytes(),
           EnumSet.of(XAttrSetFlag.CREATE));
       }
@@ -1080,6 +1105,7 @@ public class TestRetryCacheWithHA {
 
     @Override
     void invoke() throws Exception {
+      expectedUpdateCount++;
       client.removeXAttr(src, "user.key");
     }
 
@@ -1316,6 +1342,13 @@ public class TestRetryCacheWithHA {
     assertTrue("CacheUpdated on NN0: " + updatedNN0, updatedNN0 > 0);
     // Cache updated metrics on NN0 should be >0 since NN1 applied the editlog
     assertTrue("CacheUpdated on NN1: " + updatedNN1, updatedNN1 > 0);
+    long expectedUpdateCount = op.getExpectedCacheUpdateCount();
+    if (expectedUpdateCount > 0) {
+      assertEquals("CacheUpdated on NN0: " + updatedNN0, expectedUpdateCount,
+          updatedNN0);
+      assertEquals("CacheUpdated on NN0: " + updatedNN1, expectedUpdateCount,
+          updatedNN1);
+    }
   }
 
   /**


[25/50] [abbrv] hadoop git commit: HADOOP-11930. test-patch in offline mode should tell maven to be in offline mode (Sean Busbey via aw)

Posted by zj...@apache.org.
HADOOP-11930. test-patch in offline mode should tell maven to be in offline mode (Sean Busbey via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b4aeba0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b4aeba0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b4aeba0

Branch: refs/heads/YARN-2928
Commit: 4b4aeba0db26dae2d6449b33714fcb5fc5bb7efd
Parents: 81f0adb
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 12:11:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700

----------------------------------------------------------------------
 dev-support/test-patch.sh                       | 35 ++++++++++++--------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 ++
 2 files changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4aeba0/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index f760951..1409467 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -33,6 +33,8 @@ function setup_defaults
   else
     MVN=${MAVEN_HOME}/bin/mvn
   fi
+  # This parameter needs to be kept as an array
+  MAVEN_ARGS=()
 
   PROJECT_NAME=hadoop
   HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
@@ -756,6 +758,11 @@ function parse_args
     esac
   done
 
+  # if we requested offline, pass that to mvn
+  if [[ ${OFFLINE} == "true" ]] ; then
+    MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline)
+  fi
+
   # we need absolute dir for ${BASEDIR}
   cd "${CWD}"
   BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P)
@@ -1027,7 +1034,7 @@ function precheck_without_patch
 
   if [[ $? == 1 ]]; then
     echo "Compiling ${mypwd}"
-    echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
+    echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
     if [[ $? != 0 ]] ; then
       echo "${PATCH_BRANCH} compilation is broken?"
       add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken."
@@ -1041,7 +1048,7 @@ function precheck_without_patch
 
   if [[ $? == 1 ]]; then
     echo "Javadoc'ing ${mypwd}"
-    echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
+    echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
     if [[ $? != 0 ]] ; then
       echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?"
       add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken."
@@ -1055,7 +1062,7 @@ function precheck_without_patch
 
   if [[ $? == 1 ]]; then
     echo "site creation for ${mypwd}"
-    echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
+    echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
     if [[ $? != 0 ]] ; then
       echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?"
       add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken."
@@ -1633,12 +1640,12 @@ function check_javadoc
   start_clock
 
   if [[ -d hadoop-project ]]; then
-    (cd hadoop-project; "${MVN}" install > /dev/null 2>&1)
+    (cd hadoop-project; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
   fi
   if [[ -d hadoop-common-project/hadoop-annotations ]]; then
-    (cd hadoop-common-project/hadoop-annotations; "${MVN}" install > /dev/null 2>&1)
+    (cd hadoop-common-project/hadoop-annotations; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
   fi
-  echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt"  "${MVN}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
+  echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt"  "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
   count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt"
   numBranchJavadocWarnings=$?
   count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt"
@@ -1688,7 +1695,7 @@ function check_site
   start_clock
 
   echo "site creation for ${mypwd}"
-  echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
+  echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
   if [[ $? != 0 ]] ; then
     echo "Site compilation is broken"
     add_jira_table -1 site "Site compilation is broken."
@@ -1734,7 +1741,7 @@ function check_javac
 
   start_clock
 
-  echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch
+  echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch
   if [[ $? != 0 ]] ; then
     add_jira_table -1 javac "The patch appears to cause the build to fail."
     return 2
@@ -1784,7 +1791,7 @@ function check_apachelicense
 
   start_clock
 
-  echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" apache-rat:check -D${PROJECT_NAME}PatchProcess
+  echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" apache-rat:check -D${PROJECT_NAME}PatchProcess
   #shellcheck disable=SC2038
   find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt"
 
@@ -1839,7 +1846,7 @@ function check_mvn_install
   big_console_header "Installing all of the jars"
 
   start_clock
-  echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess
+  echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" "${MAVEN_ARGS[@]}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess
   retval=$?
   if [[ ${retval} != 0 ]]; then
     add_jira_table -1 install "The patch causes mvn install to fail."
@@ -1877,7 +1884,7 @@ function findbugs_mvnrunner
   local logfile=$2
   local warnings_file=$3
 
-  echo_and_redirect "${logfile}" "${MVN}" clean test findbugs:findbugs -DskipTests \
+  echo_and_redirect "${logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean test findbugs:findbugs -DskipTests \
     "-D${PROJECT_NAME}PatchProcess" < /dev/null
   if [[ $? != 0 ]]; then
     return 1
@@ -2120,7 +2127,7 @@ function check_mvn_eclipse
 
   start_clock
 
-  echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess
+  echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess
   if [[ $? != 0 ]] ; then
     add_jira_table -1 eclipse:eclipse "The patch failed to build with eclipse:eclipse."
     return 1
@@ -2211,7 +2218,7 @@ function check_unittests
     ordered_modules="${ordered_modules} ${hdfs_modules}"
     if [[ ${building_common} -eq 0 ]]; then
       echo "  Building hadoop-common with -Pnative in order to provide libhadoop.so to the hadoop-hdfs unit tests."
-      echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess"
+      echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" "${MAVEN_ARGS[@]}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess"
       if [[ $? != 0 ]]; then
         add_jira_table -1 "native" "Failed to build the native portion " \
           "of hadoop-common prior to running the unit tests in ${ordered_modules}"
@@ -2231,7 +2238,7 @@ function check_unittests
 
     test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt
     echo "  Running tests in ${module_suffix}"
-    echo_and_redirect "${test_logfile}" "${MVN}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} -D${PROJECT_NAME}PatchProcess
+    echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} -D${PROJECT_NAME}PatchProcess
     test_build_result=$?
 
     add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4aeba0/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2b85c1c..6aa224c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -788,6 +788,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8429. Avoid stuck threads if there is an error in DomainSocketWatcher
     that stops the thread.  (zhouyingchao via cmccabe)
 
+    HADOOP-11930. test-patch in offline mode should tell maven to be in
+    offline mode (Sean Busbey via aw)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES


[19/50] [abbrv] hadoop git commit: HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating (Masatake Iwasaki via Colin P. McCabe)

Posted by zj...@apache.org.
HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating (Masatake Iwasaki via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81f0adb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81f0adb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81f0adb7

Branch: refs/heads/YARN-2928
Commit: 81f0adb7c4c8b2310e2bba02643b76544a0d1d9c
Parents: 3d5c0d4
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu May 28 12:00:55 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 ++
 .../apache/hadoop/tracing/SpanReceiverHost.java | 45 ++------------------
 .../hadoop-common/src/site/markdown/Tracing.md  | 12 +++---
 .../java/org/apache/hadoop/hdfs/DFSClient.java  | 16 ++-----
 hadoop-project/pom.xml                          |  2 +-
 5 files changed, 18 insertions(+), 60 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 51eff78..2b85c1c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -620,6 +620,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12030. test-patch should only report on newly introduced
     findbugs warnings. (Sean Busbey via aw)
 
+    HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating
+    (Masatake Iwasaki via Colin P. McCabe)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
index bf9479b..9a99e05 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
@@ -17,33 +17,23 @@
  */
 package org.apache.hadoop.tracing;
 
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
-import java.util.UUID;
 
-import org.apache.commons.io.Charsets;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.tracing.SpanReceiverInfo.ConfigurationPair;
-import org.apache.hadoop.tracing.TraceUtils;
-import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.htrace.SpanReceiver;
 import org.apache.htrace.SpanReceiverBuilder;
 import org.apache.htrace.Trace;
+import org.apache.htrace.impl.LocalFileSpanReceiver;
 
 /**
  * This class provides functions for reading the names of SpanReceivers from
@@ -54,7 +44,7 @@ import org.apache.htrace.Trace;
 @InterfaceAudience.Private
 public class SpanReceiverHost implements TraceAdminProtocol {
   public static final String SPAN_RECEIVERS_CONF_SUFFIX =
-    "spanreceiver.classes";
+      "spanreceiver.classes";
   private static final Log LOG = LogFactory.getLog(SpanReceiverHost.class);
   private static final HashMap<String, SpanReceiverHost> hosts =
       new HashMap<String, SpanReceiverHost>(1);
@@ -88,33 +78,6 @@ public class SpanReceiverHost implements TraceAdminProtocol {
 
   private static List<ConfigurationPair> EMPTY = Collections.emptyList();
 
-  private static String getUniqueLocalTraceFileName() {
-    String tmp = System.getProperty("java.io.tmpdir", "/tmp");
-    String nonce = null;
-    BufferedReader reader = null;
-    try {
-      // On Linux we can get a unique local file name by reading the process id
-      // out of /proc/self/stat.  (There isn't any portable way to get the
-      // process ID from Java.)
-      reader = new BufferedReader(
-          new InputStreamReader(new FileInputStream("/proc/self/stat"),
-                                Charsets.UTF_8));
-      String line = reader.readLine();
-      if (line == null) {
-        throw new EOFException();
-      }
-      nonce = line.split(" ")[0];
-    } catch (IOException e) {
-    } finally {
-      IOUtils.cleanup(LOG, reader);
-    }
-    if (nonce == null) {
-      // If we can't use the process ID, use a random nonce.
-      nonce = UUID.randomUUID().toString();
-    }
-    return new File(tmp, nonce).getAbsolutePath();
-  }
-
   private SpanReceiverHost(String confPrefix) {
     this.confPrefix = confPrefix;
   }
@@ -143,7 +106,7 @@ public class SpanReceiverHost implements TraceAdminProtocol {
     // testing.
     String pathKey = confPrefix + LOCAL_FILE_SPAN_RECEIVER_PATH_SUFFIX;
     if (config.get(pathKey) == null) {
-      String uniqueFile = getUniqueLocalTraceFileName();
+      String uniqueFile = LocalFileSpanReceiver.getUniqueLocalTraceFileName();
       config.set(pathKey, uniqueFile);
       if (LOG.isTraceEnabled()) {
         LOG.trace("Set " + pathKey + " to " + uniqueFile);
@@ -191,7 +154,7 @@ public class SpanReceiverHost implements TraceAdminProtocol {
 
   public synchronized SpanReceiverInfo[] listSpanReceivers()
       throws IOException {
-    SpanReceiverInfo info[] = new SpanReceiverInfo[receivers.size()];
+    SpanReceiverInfo[] info = new SpanReceiverInfo[receivers.size()];
     int i = 0;
 
     for(Map.Entry<Long, SpanReceiver> entry : receivers.entrySet()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
index 76b5ee0..bf5685c 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
@@ -78,6 +78,10 @@ You also need to add the jar bundling SpanReceiver to the classpath of Hadoop
 on each node. (LocalFileSpanReceiver in the example above is included in the
 jar of htrace-core which is bundled with Hadoop.)
 
+```
+    $ cp htrace-htraced/target/htrace-htraced-3.2.0-incubating.jar $HADOOP_HOME/share/hadoop/common/lib/
+```
+
 ### Dynamic update of tracing configuration
 
 You can use `hadoop trace` command to see and update the tracing configuration of each servers.
@@ -149,6 +153,7 @@ which start tracing span before invoking HDFS shell command.
     import org.apache.hadoop.conf.Configuration;
     import org.apache.hadoop.fs.FsShell;
     import org.apache.hadoop.hdfs.DFSConfigKeys;
+    import org.apache.hadoop.hdfs.HdfsConfiguration;
     import org.apache.hadoop.tracing.SpanReceiverHost;
     import org.apache.hadoop.util.ToolRunner;
     import org.apache.htrace.Sampler;
@@ -157,19 +162,16 @@ which start tracing span before invoking HDFS shell command.
 
     public class TracingFsShell {
       public static void main(String argv[]) throws Exception {
-        Configuration conf = new Configuration();
+        Configuration conf = new HdfsConfiguration();
         FsShell shell = new FsShell();
         conf.setQuietMode(false);
         shell.setConf(conf);
         SpanReceiverHost.get(conf, DFSConfigKeys.DFS_SERVER_HTRACE_PREFIX);
         int res = 0;
-        TraceScope ts = null;
-        try {
-          ts = Trace.startSpan("FsShell", Sampler.ALWAYS);
+        try (TraceScope ts = Trace.startSpan("FsShell", Sampler.ALWAYS)) {
           res = ToolRunner.run(shell, argv);
         } finally {
           shell.close();
-          if (ts != null) ts.close();
         }
         System.exit(res);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 60e5577..fc1cd26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -37,7 +37,6 @@ import java.net.Socket;
 import java.net.SocketAddress;
 import java.net.URI;
 import java.net.UnknownHostException;
-import java.nio.charset.Charset;
 import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -3223,35 +3222,26 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
     return saslClient;
   }
 
-  private static final byte[] PATH = "path".getBytes(Charset.forName("UTF-8"));
-
   TraceScope getPathTraceScope(String description, String path) {
     TraceScope scope = Trace.startSpan(description, traceSampler);
     Span span = scope.getSpan();
     if (span != null) {
       if (path != null) {
-        span.addKVAnnotation(PATH,
-            path.getBytes(Charset.forName("UTF-8")));
+        span.addKVAnnotation("path", path);
       }
     }
     return scope;
   }
 
-  private static final byte[] SRC = "src".getBytes(Charset.forName("UTF-8"));
-
-  private static final byte[] DST = "dst".getBytes(Charset.forName("UTF-8"));
-
   TraceScope getSrcDstTraceScope(String description, String src, String dst) {
     TraceScope scope = Trace.startSpan(description, traceSampler);
     Span span = scope.getSpan();
     if (span != null) {
       if (src != null) {
-        span.addKVAnnotation(SRC,
-            src.getBytes(Charset.forName("UTF-8")));
+        span.addKVAnnotation("src", src);
       }
       if (dst != null) {
-        span.addKVAnnotation(DST,
-            dst.getBytes(Charset.forName("UTF-8")));
+        span.addKVAnnotation("dst", dst);
       }
     }
     return scope;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 73e29f8..94e1f5e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -820,7 +820,7 @@
       <dependency>
         <groupId>org.apache.htrace</groupId>
         <artifactId>htrace-core</artifactId>
-        <version>3.1.0-incubating</version>
+        <version>3.2.0-incubating</version>
       </dependency>
       <dependency>
         <groupId>org.jdom</groupId>


[05/50] [abbrv] hadoop git commit: HDFS-8482. Rename BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.

Posted by zj...@apache.org.
HDFS-8482. Rename BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b88df72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b88df72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b88df72

Branch: refs/heads/YARN-2928
Commit: 8b88df723b75e8a81ed7db7599baeb4098059f46
Parents: 00c4b9d
Author: Andrew Wang <wa...@apache.org>
Authored: Wed May 27 15:37:01 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 15:46:59 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   2 +
 .../server/blockmanagement/BlockCollection.java |   8 +-
 .../hdfs/server/blockmanagement/BlockInfo.java  | 399 +++++++++++++++++++
 .../blockmanagement/BlockInfoContiguous.java    | 399 -------------------
 .../BlockInfoContiguousUnderConstruction.java   |   6 +-
 .../server/blockmanagement/BlockManager.java    | 124 +++---
 .../hdfs/server/blockmanagement/BlocksMap.java  |  32 +-
 .../CacheReplicationMonitor.java                |  14 +-
 .../blockmanagement/DatanodeDescriptor.java     |  16 +-
 .../blockmanagement/DatanodeStorageInfo.java    |  25 +-
 .../blockmanagement/DecommissionManager.java    |  22 +-
 .../hdfs/server/namenode/FSDirWriteFileOp.java  |  14 +-
 .../hdfs/server/namenode/FSDirectory.java       |   6 +-
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |   8 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |  14 +-
 .../hdfs/server/namenode/FSImageFormat.java     |  14 +-
 .../server/namenode/FSImageFormatPBINode.java   |  11 +-
 .../server/namenode/FSImageSerialization.java   |   6 +-
 .../hdfs/server/namenode/FSNamesystem.java      |  42 +-
 .../namenode/FileUnderConstructionFeature.java  |   6 +-
 .../hadoop/hdfs/server/namenode/INodeFile.java  |  74 ++--
 .../hdfs/server/namenode/LeaseManager.java      |   6 +-
 .../hdfs/server/namenode/NamenodeFsck.java      |   4 +-
 .../snapshot/FSImageFormatPBSnapshot.java       |   8 +-
 .../hdfs/server/namenode/snapshot/FileDiff.java |  10 +-
 .../server/namenode/snapshot/FileDiffList.java  |  20 +-
 .../snapshot/FileWithSnapshotFeature.java       |   4 +-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java     |   4 +-
 .../apache/hadoop/hdfs/TestDecommission.java    |   4 +-
 .../server/blockmanagement/TestBlockInfo.java   |  20 +-
 .../blockmanagement/TestBlockManager.java       |  36 +-
 .../blockmanagement/TestDatanodeDescriptor.java |   4 +-
 .../blockmanagement/TestPendingReplication.java |   4 +-
 .../blockmanagement/TestReplicationPolicy.java  |   9 +-
 .../hdfs/server/namenode/CreateEditsLog.java    |   8 +-
 .../hdfs/server/namenode/TestAddBlock.java      |  12 +-
 .../namenode/TestBlockUnderConstruction.java    |   6 +-
 .../TestCommitBlockSynchronization.java         |   8 +-
 .../hdfs/server/namenode/TestEditLog.java       |   4 +-
 .../hdfs/server/namenode/TestFSImage.java       |   4 +-
 .../hadoop/hdfs/server/namenode/TestFsck.java   |   4 +-
 .../server/namenode/TestGetBlockLocations.java  |   4 +-
 .../hdfs/server/namenode/TestINodeFile.java     |   4 +-
 .../namenode/TestTruncateQuotaUpdate.java       |  16 +-
 .../namenode/snapshot/SnapshotTestHelper.java   |   4 +-
 .../snapshot/TestFileWithSnapshotFeature.java   |   6 +-
 .../snapshot/TestSnapshotBlocksMap.java         |  24 +-
 .../namenode/snapshot/TestSnapshotDeletion.java |  16 +-
 48 files changed, 745 insertions(+), 750 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ce97552..948a516 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -580,6 +580,8 @@ Release 2.8.0 - UNRELEASED
 
     HDFS-8377. Support HTTP/2 in datanode. (Duo Zhang via wheat9)
 
+    HDFS-8482. Rename BlockInfoContiguous to BlockInfo. (Zhe Zhang via wang)
+
   OPTIMIZATIONS
 
     HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
index c0a959c..02a1d05 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
@@ -31,7 +31,7 @@ public interface BlockCollection {
   /**
    * Get the last block of the collection.
    */
-  public BlockInfoContiguous getLastBlock();
+  public BlockInfo getLastBlock();
 
   /** 
    * Get content summary.
@@ -46,7 +46,7 @@ public interface BlockCollection {
   /**
    * Get the blocks.
    */
-  public BlockInfoContiguous[] getBlocks();
+  public BlockInfo[] getBlocks();
 
   /**
    * Get preferred block size for the collection 
@@ -73,13 +73,13 @@ public interface BlockCollection {
   /**
    * Set the block at the given index.
    */
-  public void setBlock(int index, BlockInfoContiguous blk);
+  public void setBlock(int index, BlockInfo blk);
 
   /**
    * Convert the last block of the collection to an under-construction block
    * and set the locations.
    */
-  public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfoContiguous lastBlock,
+  public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfo lastBlock,
       DatanodeStorageInfo[] targets) throws IOException;
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
new file mode 100644
index 0000000..0370c57
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -0,0 +1,399 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.LinkedList;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.util.LightWeightGSet;
+
+/**
+ * BlockInfo class maintains for a given block
+ * the {@link BlockCollection} it is part of and datanodes where the replicas of
+ * the block are stored.
+ */
+@InterfaceAudience.Private
+public class BlockInfo extends Block
+    implements LightWeightGSet.LinkedElement {
+  public static final BlockInfo[] EMPTY_ARRAY = {};
+
+  private BlockCollection bc;
+
+  /** For implementing {@link LightWeightGSet.LinkedElement} interface */
+  private LightWeightGSet.LinkedElement nextLinkedElement;
+
+  /**
+   * This array contains triplets of references. For each i-th storage, the
+   * block belongs to triplets[3*i] is the reference to the
+   * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
+   * references to the previous and the next blocks, respectively, in the list
+   * of blocks belonging to this storage.
+   *
+   * Using previous and next in Object triplets is done instead of a
+   * {@link LinkedList} list to efficiently use memory. With LinkedList the cost
+   * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
+   * bytes using the triplets.
+   */
+  private Object[] triplets;
+
+  /**
+   * Construct an entry for blocksmap
+   * @param replication the block's replication factor
+   */
+  public BlockInfo(short replication) {
+    this.triplets = new Object[3*replication];
+    this.bc = null;
+  }
+
+  public BlockInfo(Block blk, short replication) {
+    super(blk);
+    this.triplets = new Object[3*replication];
+    this.bc = null;
+  }
+
+  /**
+   * Copy construction.
+   * This is used to convert BlockInfoUnderConstruction
+   * @param from BlockInfo to copy from.
+   */
+  protected BlockInfo(BlockInfo from) {
+    super(from);
+    this.triplets = new Object[from.triplets.length];
+    this.bc = from.bc;
+  }
+
+  public BlockCollection getBlockCollection() {
+    return bc;
+  }
+
+  public void setBlockCollection(BlockCollection bc) {
+    this.bc = bc;
+  }
+
+  public boolean isDeleted() {
+    return (bc == null);
+  }
+
+  public DatanodeDescriptor getDatanode(int index) {
+    DatanodeStorageInfo storage = getStorageInfo(index);
+    return storage == null ? null : storage.getDatanodeDescriptor();
+  }
+
+  DatanodeStorageInfo getStorageInfo(int index) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
+    return (DatanodeStorageInfo)triplets[index*3];
+  }
+
+  private BlockInfo getPrevious(int index) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
+    BlockInfo info = (BlockInfo)triplets[index*3+1];
+    assert info == null ||
+        info.getClass().getName().startsWith(BlockInfo.class.getName()) :
+              "BlockInfo is expected at " + index*3;
+    return info;
+  }
+
+  BlockInfo getNext(int index) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
+    BlockInfo info = (BlockInfo)triplets[index*3+2];
+    assert info == null || info.getClass().getName().startsWith(
+        BlockInfo.class.getName()) :
+        "BlockInfo is expected at " + index*3;
+    return info;
+  }
+
+  private void setStorageInfo(int index, DatanodeStorageInfo storage) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
+    triplets[index*3] = storage;
+  }
+
+  /**
+   * Return the previous block on the block list for the datanode at
+   * position index. Set the previous block on the list to "to".
+   *
+   * @param index - the datanode index
+   * @param to - block to be set to previous on the list of blocks
+   * @return current previous block on the list of blocks
+   */
+  private BlockInfo setPrevious(int index, BlockInfo to) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
+    BlockInfo info = (BlockInfo)triplets[index*3+1];
+    triplets[index*3+1] = to;
+    return info;
+  }
+
+  /**
+   * Return the next block on the block list for the datanode at
+   * position index. Set the next block on the list to "to".
+   *
+   * @param index - the datanode index
+   * @param to - block to be set to next on the list of blocks
+   *    * @return current next block on the list of blocks
+   */
+  private BlockInfo setNext(int index, BlockInfo to) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
+    BlockInfo info = (BlockInfo)triplets[index*3+2];
+    triplets[index*3+2] = to;
+    return info;
+  }
+
+  public int getCapacity() {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+    return triplets.length / 3;
+  }
+
+  /**
+   * Ensure that there is enough  space to include num more triplets.
+   * @return first free triplet index.
+   */
+  private int ensureCapacity(int num) {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    int last = numNodes();
+    if(triplets.length >= (last+num)*3)
+      return last;
+    /* Not enough space left. Create a new array. Should normally
+     * happen only when replication is manually increased by the user. */
+    Object[] old = triplets;
+    triplets = new Object[(last+num)*3];
+    System.arraycopy(old, 0, triplets, 0, last*3);
+    return last;
+  }
+
+  /**
+   * Count the number of data-nodes the block belongs to.
+   */
+  public int numNodes() {
+    assert this.triplets != null : "BlockInfo is not initialized";
+    assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+    for(int idx = getCapacity()-1; idx >= 0; idx--) {
+      if(getDatanode(idx) != null)
+        return idx+1;
+    }
+    return 0;
+  }
+
+  /**
+   * Add a {@link DatanodeStorageInfo} location for a block
+   */
+  boolean addStorage(DatanodeStorageInfo storage) {
+    // find the last null node
+    int lastNode = ensureCapacity(1);
+    setStorageInfo(lastNode, storage);
+    setNext(lastNode, null);
+    setPrevious(lastNode, null);
+    return true;
+  }
+
+  /**
+   * Remove {@link DatanodeStorageInfo} location for a block
+   */
+  boolean removeStorage(DatanodeStorageInfo storage) {
+    int dnIndex = findStorageInfo(storage);
+    if(dnIndex < 0) // the node is not found
+      return false;
+    assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+      "Block is still in the list and must be removed first.";
+    // find the last not null node
+    int lastNode = numNodes()-1;
+    // replace current node triplet by the lastNode one
+    setStorageInfo(dnIndex, getStorageInfo(lastNode));
+    setNext(dnIndex, getNext(lastNode));
+    setPrevious(dnIndex, getPrevious(lastNode));
+    // set the last triplet to null
+    setStorageInfo(lastNode, null);
+    setNext(lastNode, null);
+    setPrevious(lastNode, null);
+    return true;
+  }
+
+  /**
+   * Find specified DatanodeStorageInfo.
+   * @return DatanodeStorageInfo or null if not found.
+   */
+  DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
+    int len = getCapacity();
+    for(int idx = 0; idx < len; idx++) {
+      DatanodeStorageInfo cur = getStorageInfo(idx);
+      if(cur == null)
+        break;
+      if(cur.getDatanodeDescriptor() == dn)
+        return cur;
+    }
+    return null;
+  }
+
+  /**
+   * Find specified DatanodeStorageInfo.
+   * @return index or -1 if not found.
+   */
+  int findStorageInfo(DatanodeStorageInfo storageInfo) {
+    int len = getCapacity();
+    for(int idx = 0; idx < len; idx++) {
+      DatanodeStorageInfo cur = getStorageInfo(idx);
+      if (cur == storageInfo) {
+        return idx;
+      }
+      if (cur == null) {
+        break;
+      }
+    }
+    return -1;
+  }
+
+  /**
+   * Insert this block into the head of the list of blocks
+   * related to the specified DatanodeStorageInfo.
+   * If the head is null then form a new list.
+   * @return current block as the new head of the list.
+   */
+  BlockInfo listInsert(BlockInfo head,
+      DatanodeStorageInfo storage) {
+    int dnIndex = this.findStorageInfo(storage);
+    assert dnIndex >= 0 : "Data node is not found: current";
+    assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+            "Block is already in the list and cannot be inserted.";
+    this.setPrevious(dnIndex, null);
+    this.setNext(dnIndex, head);
+    if(head != null)
+      head.setPrevious(head.findStorageInfo(storage), this);
+    return this;
+  }
+
+  /**
+   * Remove this block from the list of blocks
+   * related to the specified DatanodeStorageInfo.
+   * If this block is the head of the list then return the next block as
+   * the new head.
+   * @return the new head of the list or null if the list becomes
+   * empy after deletion.
+   */
+  BlockInfo listRemove(BlockInfo head,
+      DatanodeStorageInfo storage) {
+    if(head == null)
+      return null;
+    int dnIndex = this.findStorageInfo(storage);
+    if(dnIndex < 0) // this block is not on the data-node list
+      return head;
+
+    BlockInfo next = this.getNext(dnIndex);
+    BlockInfo prev = this.getPrevious(dnIndex);
+    this.setNext(dnIndex, null);
+    this.setPrevious(dnIndex, null);
+    if(prev != null)
+      prev.setNext(prev.findStorageInfo(storage), next);
+    if(next != null)
+      next.setPrevious(next.findStorageInfo(storage), prev);
+    if(this == head)  // removing the head
+      head = next;
+    return head;
+  }
+
+  /**
+   * Remove this block from the list of blocks related to the specified
+   * DatanodeDescriptor. Insert it into the head of the list of blocks.
+   *
+   * @return the new head of the list.
+   */
+  public BlockInfo moveBlockToHead(BlockInfo head,
+      DatanodeStorageInfo storage, int curIndex, int headIndex) {
+    if (head == this) {
+      return this;
+    }
+    BlockInfo next = this.setNext(curIndex, head);
+    BlockInfo prev = this.setPrevious(curIndex, null);
+
+    head.setPrevious(headIndex, this);
+    prev.setNext(prev.findStorageInfo(storage), next);
+    if (next != null) {
+      next.setPrevious(next.findStorageInfo(storage), prev);
+    }
+    return this;
+  }
+
+  /**
+   * BlockInfo represents a block that is not being constructed.
+   * In order to start modifying the block, the BlockInfo should be converted
+   * to {@link BlockInfoContiguousUnderConstruction}.
+   * @return {@link BlockUCState#COMPLETE}
+   */
+  public BlockUCState getBlockUCState() {
+    return BlockUCState.COMPLETE;
+  }
+
+  /**
+   * Is this block complete?
+   *
+   * @return true if the state of the block is {@link BlockUCState#COMPLETE}
+   */
+  public boolean isComplete() {
+    return getBlockUCState().equals(BlockUCState.COMPLETE);
+  }
+
+  /**
+   * Convert a complete block to an under construction block.
+   * @return BlockInfoUnderConstruction -  an under construction block.
+   */
+  public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
+      BlockUCState s, DatanodeStorageInfo[] targets) {
+    if(isComplete()) {
+      BlockInfoContiguousUnderConstruction ucBlock =
+          new BlockInfoContiguousUnderConstruction(this,
+          getBlockCollection().getPreferredBlockReplication(), s, targets);
+      ucBlock.setBlockCollection(getBlockCollection());
+      return ucBlock;
+    }
+    // the block is already under construction
+    BlockInfoContiguousUnderConstruction ucBlock =
+        (BlockInfoContiguousUnderConstruction)this;
+    ucBlock.setBlockUCState(s);
+    ucBlock.setExpectedLocations(targets);
+    ucBlock.setBlockCollection(getBlockCollection());
+    return ucBlock;
+  }
+
+  @Override
+  public int hashCode() {
+    // Super implementation is sufficient
+    return super.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    // Sufficient to rely on super's implementation
+    return (this == obj) || super.equals(obj);
+  }
+
+  @Override
+  public LightWeightGSet.LinkedElement getNext() {
+    return nextLinkedElement;
+  }
+
+  @Override
+  public void setNext(LightWeightGSet.LinkedElement next) {
+    this.nextLinkedElement = next;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
deleted file mode 100644
index 769046b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ /dev/null
@@ -1,399 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.blockmanagement;
-
-import java.util.LinkedList;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import org.apache.hadoop.util.LightWeightGSet;
-
-/**
- * BlockInfo class maintains for a given block
- * the {@link BlockCollection} it is part of and datanodes where the replicas of 
- * the block are stored.
- */
-@InterfaceAudience.Private
-public class BlockInfoContiguous extends Block
-    implements LightWeightGSet.LinkedElement {
-  public static final BlockInfoContiguous[] EMPTY_ARRAY = {};
-
-  private BlockCollection bc;
-
-  /** For implementing {@link LightWeightGSet.LinkedElement} interface */
-  private LightWeightGSet.LinkedElement nextLinkedElement;
-
-  /**
-   * This array contains triplets of references. For each i-th storage, the
-   * block belongs to triplets[3*i] is the reference to the
-   * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
-   * references to the previous and the next blocks, respectively, in the list
-   * of blocks belonging to this storage.
-   * 
-   * Using previous and next in Object triplets is done instead of a
-   * {@link LinkedList} list to efficiently use memory. With LinkedList the cost
-   * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
-   * bytes using the triplets.
-   */
-  private Object[] triplets;
-
-  /**
-   * Construct an entry for blocksmap
-   * @param replication the block's replication factor
-   */
-  public BlockInfoContiguous(short replication) {
-    this.triplets = new Object[3*replication];
-    this.bc = null;
-  }
-  
-  public BlockInfoContiguous(Block blk, short replication) {
-    super(blk);
-    this.triplets = new Object[3*replication];
-    this.bc = null;
-  }
-
-  /**
-   * Copy construction.
-   * This is used to convert BlockInfoUnderConstruction
-   * @param from BlockInfo to copy from.
-   */
-  protected BlockInfoContiguous(BlockInfoContiguous from) {
-    super(from);
-    this.triplets = new Object[from.triplets.length];
-    this.bc = from.bc;
-  }
-
-  public BlockCollection getBlockCollection() {
-    return bc;
-  }
-
-  public void setBlockCollection(BlockCollection bc) {
-    this.bc = bc;
-  }
-
-  public boolean isDeleted() {
-    return (bc == null);
-  }
-
-  public DatanodeDescriptor getDatanode(int index) {
-    DatanodeStorageInfo storage = getStorageInfo(index);
-    return storage == null ? null : storage.getDatanodeDescriptor();
-  }
-
-  DatanodeStorageInfo getStorageInfo(int index) {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
-    return (DatanodeStorageInfo)triplets[index*3];
-  }
-
-  private BlockInfoContiguous getPrevious(int index) {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
-    BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
-    assert info == null || 
-        info.getClass().getName().startsWith(BlockInfoContiguous.class.getName()) :
-              "BlockInfo is expected at " + index*3;
-    return info;
-  }
-
-  BlockInfoContiguous getNext(int index) {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
-    BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
-    assert info == null || info.getClass().getName().startsWith(
-        BlockInfoContiguous.class.getName()) :
-        "BlockInfo is expected at " + index*3;
-    return info;
-  }
-
-  private void setStorageInfo(int index, DatanodeStorageInfo storage) {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
-    triplets[index*3] = storage;
-  }
-
-  /**
-   * Return the previous block on the block list for the datanode at
-   * position index. Set the previous block on the list to "to".
-   *
-   * @param index - the datanode index
-   * @param to - block to be set to previous on the list of blocks
-   * @return current previous block on the list of blocks
-   */
-  private BlockInfoContiguous setPrevious(int index, BlockInfoContiguous to) {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
-    BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
-    triplets[index*3+1] = to;
-    return info;
-  }
-
-  /**
-   * Return the next block on the block list for the datanode at
-   * position index. Set the next block on the list to "to".
-   *
-   * @param index - the datanode index
-   * @param to - block to be set to next on the list of blocks
-   *    * @return current next block on the list of blocks
-   */
-  private BlockInfoContiguous setNext(int index, BlockInfoContiguous to) {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
-    BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
-    triplets[index*3+2] = to;
-    return info;
-  }
-
-  public int getCapacity() {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    assert triplets.length % 3 == 0 : "Malformed BlockInfo";
-    return triplets.length / 3;
-  }
-
-  /**
-   * Ensure that there is enough  space to include num more triplets.
-   * @return first free triplet index.
-   */
-  private int ensureCapacity(int num) {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    int last = numNodes();
-    if(triplets.length >= (last+num)*3)
-      return last;
-    /* Not enough space left. Create a new array. Should normally 
-     * happen only when replication is manually increased by the user. */
-    Object[] old = triplets;
-    triplets = new Object[(last+num)*3];
-    System.arraycopy(old, 0, triplets, 0, last*3);
-    return last;
-  }
-
-  /**
-   * Count the number of data-nodes the block belongs to.
-   */
-  public int numNodes() {
-    assert this.triplets != null : "BlockInfo is not initialized";
-    assert triplets.length % 3 == 0 : "Malformed BlockInfo";
-    for(int idx = getCapacity()-1; idx >= 0; idx--) {
-      if(getDatanode(idx) != null)
-        return idx+1;
-    }
-    return 0;
-  }
-
-  /**
-   * Add a {@link DatanodeStorageInfo} location for a block
-   */
-  boolean addStorage(DatanodeStorageInfo storage) {
-    // find the last null node
-    int lastNode = ensureCapacity(1);
-    setStorageInfo(lastNode, storage);
-    setNext(lastNode, null);
-    setPrevious(lastNode, null);
-    return true;
-  }
-
-  /**
-   * Remove {@link DatanodeStorageInfo} location for a block
-   */
-  boolean removeStorage(DatanodeStorageInfo storage) {
-    int dnIndex = findStorageInfo(storage);
-    if(dnIndex < 0) // the node is not found
-      return false;
-    assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : 
-      "Block is still in the list and must be removed first.";
-    // find the last not null node
-    int lastNode = numNodes()-1; 
-    // replace current node triplet by the lastNode one 
-    setStorageInfo(dnIndex, getStorageInfo(lastNode));
-    setNext(dnIndex, getNext(lastNode)); 
-    setPrevious(dnIndex, getPrevious(lastNode)); 
-    // set the last triplet to null
-    setStorageInfo(lastNode, null);
-    setNext(lastNode, null); 
-    setPrevious(lastNode, null); 
-    return true;
-  }
-
-  /**
-   * Find specified DatanodeStorageInfo.
-   * @return DatanodeStorageInfo or null if not found.
-   */
-  DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
-    int len = getCapacity();
-    for(int idx = 0; idx < len; idx++) {
-      DatanodeStorageInfo cur = getStorageInfo(idx);
-      if(cur == null)
-        break;
-      if(cur.getDatanodeDescriptor() == dn)
-        return cur;
-    }
-    return null;
-  }
-  
-  /**
-   * Find specified DatanodeStorageInfo.
-   * @return index or -1 if not found.
-   */
-  int findStorageInfo(DatanodeStorageInfo storageInfo) {
-    int len = getCapacity();
-    for(int idx = 0; idx < len; idx++) {
-      DatanodeStorageInfo cur = getStorageInfo(idx);
-      if (cur == storageInfo) {
-        return idx;
-      }
-      if (cur == null) {
-        break;
-      }
-    }
-    return -1;
-  }
-
-  /**
-   * Insert this block into the head of the list of blocks 
-   * related to the specified DatanodeStorageInfo.
-   * If the head is null then form a new list.
-   * @return current block as the new head of the list.
-   */
-  BlockInfoContiguous listInsert(BlockInfoContiguous head,
-      DatanodeStorageInfo storage) {
-    int dnIndex = this.findStorageInfo(storage);
-    assert dnIndex >= 0 : "Data node is not found: current";
-    assert getPrevious(dnIndex) == null && getNext(dnIndex) == null : 
-            "Block is already in the list and cannot be inserted.";
-    this.setPrevious(dnIndex, null);
-    this.setNext(dnIndex, head);
-    if(head != null)
-      head.setPrevious(head.findStorageInfo(storage), this);
-    return this;
-  }
-
-  /**
-   * Remove this block from the list of blocks 
-   * related to the specified DatanodeStorageInfo.
-   * If this block is the head of the list then return the next block as 
-   * the new head.
-   * @return the new head of the list or null if the list becomes
-   * empy after deletion.
-   */
-  BlockInfoContiguous listRemove(BlockInfoContiguous head,
-      DatanodeStorageInfo storage) {
-    if(head == null)
-      return null;
-    int dnIndex = this.findStorageInfo(storage);
-    if(dnIndex < 0) // this block is not on the data-node list
-      return head;
-
-    BlockInfoContiguous next = this.getNext(dnIndex);
-    BlockInfoContiguous prev = this.getPrevious(dnIndex);
-    this.setNext(dnIndex, null);
-    this.setPrevious(dnIndex, null);
-    if(prev != null)
-      prev.setNext(prev.findStorageInfo(storage), next);
-    if(next != null)
-      next.setPrevious(next.findStorageInfo(storage), prev);
-    if(this == head)  // removing the head
-      head = next;
-    return head;
-  }
-
-  /**
-   * Remove this block from the list of blocks related to the specified
-   * DatanodeDescriptor. Insert it into the head of the list of blocks.
-   *
-   * @return the new head of the list.
-   */
-  public BlockInfoContiguous moveBlockToHead(BlockInfoContiguous head,
-      DatanodeStorageInfo storage, int curIndex, int headIndex) {
-    if (head == this) {
-      return this;
-    }
-    BlockInfoContiguous next = this.setNext(curIndex, head);
-    BlockInfoContiguous prev = this.setPrevious(curIndex, null);
-
-    head.setPrevious(headIndex, this);
-    prev.setNext(prev.findStorageInfo(storage), next);
-    if (next != null) {
-      next.setPrevious(next.findStorageInfo(storage), prev);
-    }
-    return this;
-  }
-
-  /**
-   * BlockInfo represents a block that is not being constructed.
-   * In order to start modifying the block, the BlockInfo should be converted
-   * to {@link BlockInfoContiguousUnderConstruction}.
-   * @return {@link BlockUCState#COMPLETE}
-   */
-  public BlockUCState getBlockUCState() {
-    return BlockUCState.COMPLETE;
-  }
-
-  /**
-   * Is this block complete?
-   * 
-   * @return true if the state of the block is {@link BlockUCState#COMPLETE}
-   */
-  public boolean isComplete() {
-    return getBlockUCState().equals(BlockUCState.COMPLETE);
-  }
-
-  /**
-   * Convert a complete block to an under construction block.
-   * @return BlockInfoUnderConstruction -  an under construction block.
-   */
-  public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
-      BlockUCState s, DatanodeStorageInfo[] targets) {
-    if(isComplete()) {
-      BlockInfoContiguousUnderConstruction ucBlock =
-          new BlockInfoContiguousUnderConstruction(this,
-          getBlockCollection().getPreferredBlockReplication(), s, targets);
-      ucBlock.setBlockCollection(getBlockCollection());
-      return ucBlock;
-    }
-    // the block is already under construction
-    BlockInfoContiguousUnderConstruction ucBlock =
-        (BlockInfoContiguousUnderConstruction)this;
-    ucBlock.setBlockUCState(s);
-    ucBlock.setExpectedLocations(targets);
-    ucBlock.setBlockCollection(getBlockCollection());
-    return ucBlock;
-  }
-
-  @Override
-  public int hashCode() {
-    // Super implementation is sufficient
-    return super.hashCode();
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    // Sufficient to rely on super's implementation
-    return (this == obj) || super.equals(obj);
-  }
-
-  @Override
-  public LightWeightGSet.LinkedElement getNext() {
-    return nextLinkedElement;
-  }
-
-  @Override
-  public void setNext(LightWeightGSet.LinkedElement next) {
-    this.nextLinkedElement = next;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
index 92153ab..58020a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
  * Represents a block that is currently being constructed.<br>
  * This is usually the last block of a file opened for write or append.
  */
-public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
+public class BlockInfoContiguousUnderConstruction extends BlockInfo {
   /** Block state. See {@link BlockUCState} */
   private BlockUCState blockUCState;
 
@@ -182,10 +182,10 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
    * the client or it does not have at least a minimal number of replicas 
    * reported from data-nodes. 
    */
-  BlockInfoContiguous convertToCompleteBlock() throws IOException {
+  BlockInfo convertToCompleteBlock() throws IOException {
     assert getBlockUCState() != BlockUCState.COMPLETE :
       "Trying to convert a COMPLETE block";
-    return new BlockInfoContiguous(this);
+    return new BlockInfo(this);
   }
 
   /** Set expected locations */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 54981fb..45a485b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -543,8 +543,8 @@ public class BlockManager {
     int usableReplicas = numReplicas.liveReplicas() +
                          numReplicas.decommissionedAndDecommissioning();
     
-    if (block instanceof BlockInfoContiguous) {
-      BlockCollection bc = ((BlockInfoContiguous) block).getBlockCollection();
+    if (block instanceof BlockInfo) {
+      BlockCollection bc = ((BlockInfo) block).getBlockCollection();
       String fileName = (bc == null) ? "[orphaned]" : bc.getName();
       out.print(fileName + ": ");
     }
@@ -624,7 +624,7 @@ public class BlockManager {
       Block commitBlock) throws IOException {
     if(commitBlock == null)
       return false; // not committing, this is a block allocation retry
-    BlockInfoContiguous lastBlock = bc.getLastBlock();
+    BlockInfo lastBlock = bc.getLastBlock();
     if(lastBlock == null)
       return false; // no blocks in file yet
     if(lastBlock.isComplete())
@@ -644,11 +644,11 @@ public class BlockManager {
    * @throws IOException if the block does not have at least a minimal number
    * of replicas reported from data-nodes.
    */
-  private BlockInfoContiguous completeBlock(final BlockCollection bc,
+  private BlockInfo completeBlock(final BlockCollection bc,
       final int blkIndex, boolean force) throws IOException {
     if(blkIndex < 0)
       return null;
-    BlockInfoContiguous curBlock = bc.getBlocks()[blkIndex];
+    BlockInfo curBlock = bc.getBlocks()[blkIndex];
     if(curBlock.isComplete())
       return curBlock;
     BlockInfoContiguousUnderConstruction ucBlock =
@@ -660,7 +660,7 @@ public class BlockManager {
     if(!force && ucBlock.getBlockUCState() != BlockUCState.COMMITTED)
       throw new IOException(
           "Cannot complete block: block has not been COMMITTED by the client");
-    BlockInfoContiguous completeBlock = ucBlock.convertToCompleteBlock();
+    BlockInfo completeBlock = ucBlock.convertToCompleteBlock();
     // replace penultimate block in file
     bc.setBlock(blkIndex, completeBlock);
     
@@ -678,9 +678,9 @@ public class BlockManager {
     return blocksMap.replaceBlock(completeBlock);
   }
 
-  private BlockInfoContiguous completeBlock(final BlockCollection bc,
-      final BlockInfoContiguous block, boolean force) throws IOException {
-    BlockInfoContiguous[] fileBlocks = bc.getBlocks();
+  private BlockInfo completeBlock(final BlockCollection bc,
+      final BlockInfo block, boolean force) throws IOException {
+    BlockInfo[] fileBlocks = bc.getBlocks();
     for(int idx = 0; idx < fileBlocks.length; idx++)
       if(fileBlocks[idx] == block) {
         return completeBlock(bc, idx, force);
@@ -693,7 +693,7 @@ public class BlockManager {
    * regardless of whether enough replicas are present. This is necessary
    * when tailing edit logs as a Standby.
    */
-  public BlockInfoContiguous forceCompleteBlock(final BlockCollection bc,
+  public BlockInfo forceCompleteBlock(final BlockCollection bc,
       final BlockInfoContiguousUnderConstruction block) throws IOException {
     block.commitBlock(block);
     return completeBlock(bc, block, true);
@@ -716,7 +716,7 @@ public class BlockManager {
    */
   public LocatedBlock convertLastBlockToUnderConstruction(
       BlockCollection bc, long bytesToRemove) throws IOException {
-    BlockInfoContiguous oldBlock = bc.getLastBlock();
+    BlockInfo oldBlock = bc.getLastBlock();
     if(oldBlock == null ||
        bc.getPreferredBlockSize() == oldBlock.getNumBytes() - bytesToRemove)
       return null;
@@ -769,7 +769,7 @@ public class BlockManager {
   }
   
   private List<LocatedBlock> createLocatedBlockList(
-      final BlockInfoContiguous[] blocks,
+      final BlockInfo[] blocks,
       final long offset, final long length, final int nrBlocksToReturn,
       final AccessMode mode) throws IOException {
     int curBlk = 0;
@@ -799,7 +799,7 @@ public class BlockManager {
     return results;
   }
 
-  private LocatedBlock createLocatedBlock(final BlockInfoContiguous[] blocks,
+  private LocatedBlock createLocatedBlock(final BlockInfo[] blocks,
       final long endPos, final AccessMode mode) throws IOException {
     int curBlk = 0;
     long curPos = 0;
@@ -815,7 +815,7 @@ public class BlockManager {
     return createLocatedBlock(blocks[curBlk], curPos, mode);
   }
   
-  private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos,
+  private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos,
     final AccessMode mode) throws IOException {
     final LocatedBlock lb = createLocatedBlock(blk, pos);
     if (mode != null) {
@@ -825,7 +825,7 @@ public class BlockManager {
   }
 
   /** @return a LocatedBlock for the given block */
-  private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos
+  private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos
       ) throws IOException {
     if (blk instanceof BlockInfoContiguousUnderConstruction) {
       if (blk.isComplete()) {
@@ -874,7 +874,7 @@ public class BlockManager {
   }
 
   /** Create a LocatedBlocks. */
-  public LocatedBlocks createLocatedBlocks(final BlockInfoContiguous[] blocks,
+  public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks,
       final long fileSizeExcludeBlocksUnderConstruction,
       final boolean isFileUnderConstruction, final long offset,
       final long length, final boolean needBlockToken,
@@ -897,7 +897,7 @@ public class BlockManager {
       final LocatedBlock lastlb;
       final boolean isComplete;
       if (!inSnapshot) {
-        final BlockInfoContiguous last = blocks[blocks.length - 1];
+        final BlockInfo last = blocks[blocks.length - 1];
         final long lastPos = last.isComplete()?
             fileSizeExcludeBlocksUnderConstruction - last.getNumBytes()
             : fileSizeExcludeBlocksUnderConstruction;
@@ -986,7 +986,7 @@ public class BlockManager {
   /**
    * Check if a block is replicated to at least the minimum replication.
    */
-  public boolean isSufficientlyReplicated(BlockInfoContiguous b) {
+  public boolean isSufficientlyReplicated(BlockInfo b) {
     // Compare against the lesser of the minReplication and number of live DNs.
     final int replication =
         Math.min(minReplication, getDatanodeManager().getNumLiveDataNodes());
@@ -1027,7 +1027,7 @@ public class BlockManager {
     if(numBlocks == 0) {
       return new BlocksWithLocations(new BlockWithLocations[0]);
     }
-    Iterator<BlockInfoContiguous> iter = node.getBlockIterator();
+    Iterator<BlockInfo> iter = node.getBlockIterator();
     // starting from a random block
     int startBlock = ThreadLocalRandom.current().nextInt(numBlocks);
     // skip blocks
@@ -1036,7 +1036,7 @@ public class BlockManager {
     }
     List<BlockWithLocations> results = new ArrayList<BlockWithLocations>();
     long totalSize = 0;
-    BlockInfoContiguous curBlock;
+    BlockInfo curBlock;
     while(totalSize<size && iter.hasNext()) {
       curBlock = iter.next();
       if(!curBlock.isComplete())  continue;
@@ -1135,7 +1135,7 @@ public class BlockManager {
   public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk,
       final DatanodeInfo dn, String storageID, String reason) throws IOException {
     assert namesystem.hasWriteLock();
-    final BlockInfoContiguous storedBlock = getStoredBlock(blk.getLocalBlock());
+    final BlockInfo storedBlock = getStoredBlock(blk.getLocalBlock());
     if (storedBlock == null) {
       // Check if the replica is in the blockMap, if not
       // ignore the request for now. This could happen when BlockScanner
@@ -1694,7 +1694,7 @@ public class BlockManager {
            * Use the blockinfo from the blocksmap to be certain we're working
            * with the most up-to-date block information (e.g. genstamp).
            */
-          BlockInfoContiguous bi = blocksMap.getStoredBlock(timedOutItems[i]);
+          BlockInfo bi = blocksMap.getStoredBlock(timedOutItems[i]);
           if (bi == null) {
             continue;
           }
@@ -1738,16 +1738,16 @@ public class BlockManager {
    */
   private static class BlockToMarkCorrupt {
     /** The corrupted block in a datanode. */
-    final BlockInfoContiguous corrupted;
+    final BlockInfo corrupted;
     /** The corresponding block stored in the BlockManager. */
-    final BlockInfoContiguous stored;
+    final BlockInfo stored;
     /** The reason to mark corrupt. */
     final String reason;
     /** The reason code to be stored */
     final Reason reasonCode;
 
-    BlockToMarkCorrupt(BlockInfoContiguous corrupted,
-        BlockInfoContiguous stored, String reason,
+    BlockToMarkCorrupt(BlockInfo corrupted,
+        BlockInfo stored, String reason,
         Reason reasonCode) {
       Preconditions.checkNotNull(corrupted, "corrupted is null");
       Preconditions.checkNotNull(stored, "stored is null");
@@ -1758,14 +1758,14 @@ public class BlockManager {
       this.reasonCode = reasonCode;
     }
 
-    BlockToMarkCorrupt(BlockInfoContiguous stored, String reason,
+    BlockToMarkCorrupt(BlockInfo stored, String reason,
         Reason reasonCode) {
       this(stored, stored, reason, reasonCode);
     }
 
-    BlockToMarkCorrupt(BlockInfoContiguous stored, long gs, String reason,
+    BlockToMarkCorrupt(BlockInfo stored, long gs, String reason,
         Reason reasonCode) {
-      this(new BlockInfoContiguous(stored), stored, reason, reasonCode);
+      this(new BlockInfo(stored), stored, reason, reasonCode);
       //the corrupted block in datanode has a different generation stamp
       corrupted.setGenerationStamp(gs);
     }
@@ -1882,10 +1882,10 @@ public class BlockManager {
              "longer exists on the DataNode.",
               Long.toHexString(context.getReportId()), zombie.getStorageID());
     assert(namesystem.hasWriteLock());
-    Iterator<BlockInfoContiguous> iter = zombie.getBlockIterator();
+    Iterator<BlockInfo> iter = zombie.getBlockIterator();
     int prevBlocks = zombie.numBlocks();
     while (iter.hasNext()) {
-      BlockInfoContiguous block = iter.next();
+      BlockInfo block = iter.next();
       // We assume that a block can be on only one storage in a DataNode.
       // That's why we pass in the DatanodeDescriptor rather than the
       // DatanodeStorageInfo.
@@ -1943,7 +1943,7 @@ public class BlockManager {
           break;
         }
 
-        BlockInfoContiguous bi = blocksMap.getStoredBlock(b);
+        BlockInfo bi = blocksMap.getStoredBlock(b);
         if (bi == null) {
           if (LOG.isDebugEnabled()) {
             LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
@@ -1983,7 +1983,7 @@ public class BlockManager {
     // Modify the (block-->datanode) map, according to the difference
     // between the old and new block report.
     //
-    Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>();
+    Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>();
     Collection<Block> toRemove = new TreeSet<Block>();
     Collection<Block> toInvalidate = new LinkedList<Block>();
     Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>();
@@ -2000,7 +2000,7 @@ public class BlockManager {
       removeStoredBlock(b, node);
     }
     int numBlocksLogged = 0;
-    for (BlockInfoContiguous b : toAdd) {
+    for (BlockInfo b : toAdd) {
       addStoredBlock(b, storageInfo, null, numBlocksLogged < maxNumBlocksToLog);
       numBlocksLogged++;
     }
@@ -2022,7 +2022,7 @@ public class BlockManager {
    * Mark block replicas as corrupt except those on the storages in 
    * newStorages list.
    */
-  public void markBlockReplicasAsCorrupt(BlockInfoContiguous block, 
+  public void markBlockReplicasAsCorrupt(BlockInfo block,
       long oldGenerationStamp, long oldNumBytes, 
       DatanodeStorageInfo[] newStorages) throws IOException {
     assert namesystem.hasWriteLock();
@@ -2091,7 +2091,7 @@ public class BlockManager {
         continue;
       }
       
-      BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk);
+      BlockInfo storedBlock = blocksMap.getStoredBlock(iblk);
       // If block does not belong to any file, we are done.
       if (storedBlock == null) continue;
       
@@ -2136,7 +2136,7 @@ public class BlockManager {
 
   private void reportDiff(DatanodeStorageInfo storageInfo, 
       BlockListAsLongs newReport, 
-      Collection<BlockInfoContiguous> toAdd,              // add to DatanodeDescriptor
+      Collection<BlockInfo> toAdd,              // add to DatanodeDescriptor
       Collection<Block> toRemove,           // remove from DatanodeDescriptor
       Collection<Block> toInvalidate,       // should be removed from DN
       Collection<BlockToMarkCorrupt> toCorrupt, // add to corrupt replicas list
@@ -2144,7 +2144,7 @@ public class BlockManager {
 
     // place a delimiter in the list which separates blocks 
     // that have been reported from those that have not
-    BlockInfoContiguous delimiter = new BlockInfoContiguous(new Block(), (short) 1);
+    BlockInfo delimiter = new BlockInfo(new Block(), (short) 1);
     AddBlockResult result = storageInfo.addBlock(delimiter);
     assert result == AddBlockResult.ADDED 
         : "Delimiting block cannot be present in the node";
@@ -2157,7 +2157,7 @@ public class BlockManager {
     // scan the report and process newly reported blocks
     for (BlockReportReplica iblk : newReport) {
       ReplicaState iState = iblk.getState();
-      BlockInfoContiguous storedBlock = processReportedBlock(storageInfo,
+      BlockInfo storedBlock = processReportedBlock(storageInfo,
           iblk, iState, toAdd, toInvalidate, toCorrupt, toUC);
 
       // move block to the head of the list
@@ -2169,7 +2169,7 @@ public class BlockManager {
 
     // collect blocks that have not been reported
     // all of them are next to the delimiter
-    Iterator<BlockInfoContiguous> it =
+    Iterator<BlockInfo> it =
         storageInfo.new BlockIterator(delimiter.getNext(0));
     while(it.hasNext())
       toRemove.add(it.next());
@@ -2207,10 +2207,10 @@ public class BlockManager {
    * @return the up-to-date stored block, if it should be kept.
    *         Otherwise, null.
    */
-  private BlockInfoContiguous processReportedBlock(
+  private BlockInfo processReportedBlock(
       final DatanodeStorageInfo storageInfo,
       final Block block, final ReplicaState reportedState, 
-      final Collection<BlockInfoContiguous> toAdd,
+      final Collection<BlockInfo> toAdd,
       final Collection<Block> toInvalidate, 
       final Collection<BlockToMarkCorrupt> toCorrupt,
       final Collection<StatefulBlockInfo> toUC) {
@@ -2231,7 +2231,7 @@ public class BlockManager {
     }
     
     // find block by blockId
-    BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block);
+    BlockInfo storedBlock = blocksMap.getStoredBlock(block);
     if(storedBlock == null) {
       // If blocksMap does not contain reported block id,
       // the replica should be removed from the data-node.
@@ -2370,7 +2370,7 @@ public class BlockManager {
    */
   private BlockToMarkCorrupt checkReplicaCorrupt(
       Block reported, ReplicaState reportedState, 
-      BlockInfoContiguous storedBlock, BlockUCState ucState,
+      BlockInfo storedBlock, BlockUCState ucState,
       DatanodeDescriptor dn) {
     switch(reportedState) {
     case FINALIZED:
@@ -2443,7 +2443,7 @@ public class BlockManager {
     }
   }
 
-  private boolean isBlockUnderConstruction(BlockInfoContiguous storedBlock,
+  private boolean isBlockUnderConstruction(BlockInfo storedBlock,
       BlockUCState ucState, ReplicaState reportedState) {
     switch(reportedState) {
     case FINALIZED:
@@ -2487,7 +2487,7 @@ public class BlockManager {
    * 
    * @throws IOException
    */
-  private void addStoredBlockImmediate(BlockInfoContiguous storedBlock,
+  private void addStoredBlockImmediate(BlockInfo storedBlock,
       DatanodeStorageInfo storageInfo)
   throws IOException {
     assert (storedBlock != null && namesystem.hasWriteLock());
@@ -2519,13 +2519,13 @@ public class BlockManager {
    * needed replications if this takes care of the problem.
    * @return the block that is stored in blockMap.
    */
-  private Block addStoredBlock(final BlockInfoContiguous block,
+  private Block addStoredBlock(final BlockInfo block,
                                DatanodeStorageInfo storageInfo,
                                DatanodeDescriptor delNodeHint,
                                boolean logEveryBlock)
   throws IOException {
     assert block != null && namesystem.hasWriteLock();
-    BlockInfoContiguous storedBlock;
+    BlockInfo storedBlock;
     DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
     if (block instanceof BlockInfoContiguousUnderConstruction) {
       //refresh our copy in case the block got completed in another thread
@@ -2623,7 +2623,7 @@ public class BlockManager {
     return storedBlock;
   }
 
-  private void logAddStoredBlock(BlockInfoContiguous storedBlock,
+  private void logAddStoredBlock(BlockInfo storedBlock,
       DatanodeDescriptor node) {
     if (!blockLog.isInfoEnabled()) {
       return;
@@ -2651,7 +2651,7 @@ public class BlockManager {
    *
    * @param blk Block whose corrupt replicas need to be invalidated
    */
-  private void invalidateCorruptReplicas(BlockInfoContiguous blk) {
+  private void invalidateCorruptReplicas(BlockInfo blk) {
     Collection<DatanodeDescriptor> nodes = corruptReplicas.getNodes(blk);
     boolean removedFromBlocksMap = true;
     if (nodes == null)
@@ -2730,7 +2730,7 @@ public class BlockManager {
     long nrInvalid = 0, nrOverReplicated = 0;
     long nrUnderReplicated = 0, nrPostponed = 0, nrUnderConstruction = 0;
     long startTimeMisReplicatedScan = Time.monotonicNow();
-    Iterator<BlockInfoContiguous> blocksItr = blocksMap.getBlocks().iterator();
+    Iterator<BlockInfo> blocksItr = blocksMap.getBlocks().iterator();
     long totalBlocks = blocksMap.size();
     replicationQueuesInitProgress = 0;
     long totalProcessed = 0;
@@ -2742,7 +2742,7 @@ public class BlockManager {
       namesystem.writeLockInterruptibly();
       try {
         while (processed < numBlocksPerIteration && blocksItr.hasNext()) {
-          BlockInfoContiguous block = blocksItr.next();
+          BlockInfo block = blocksItr.next();
           MisReplicationResult res = processMisReplicatedBlock(block);
           if (LOG.isTraceEnabled()) {
             LOG.trace("block " + block + ": " + res);
@@ -2817,7 +2817,7 @@ public class BlockManager {
    * appropriate queues if necessary, and returns a result code indicating
    * what happened with it.
    */
-  private MisReplicationResult processMisReplicatedBlock(BlockInfoContiguous block) {
+  private MisReplicationResult processMisReplicatedBlock(BlockInfo block) {
     if (block.isDeleted()) {
       // block does not belong to any file
       addToInvalidates(block);
@@ -3157,7 +3157,7 @@ public class BlockManager {
       ReplicaState reportedState, DatanodeDescriptor delHintNode)
       throws IOException {
     // blockReceived reports a finalized block
-    Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>();
+    Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>();
     Collection<Block> toInvalidate = new LinkedList<Block>();
     Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>();
     Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>();
@@ -3174,7 +3174,7 @@ public class BlockManager {
       addStoredBlockUnderConstruction(b, storageInfo);
     }
     long numBlocksLogged = 0;
-    for (BlockInfoContiguous b : toAdd) {
+    for (BlockInfo b : toAdd) {
       addStoredBlock(b, storageInfo, delHintNode, numBlocksLogged < maxNumBlocksToLog);
       numBlocksLogged++;
     }
@@ -3301,7 +3301,7 @@ public class BlockManager {
    * @param b - the block being tested
    * @return count of live nodes for this block
    */
-  int countLiveNodes(BlockInfoContiguous b) {
+  int countLiveNodes(BlockInfo b) {
     if (!namesystem.isInStartupSafeMode()) {
       return countNodes(b).liveReplicas();
     }
@@ -3380,7 +3380,7 @@ public class BlockManager {
     return blocksMap.size();
   }
 
-  public DatanodeStorageInfo[] getStorages(BlockInfoContiguous block) {
+  public DatanodeStorageInfo[] getStorages(BlockInfo block) {
     final DatanodeStorageInfo[] storages = new DatanodeStorageInfo[block.numNodes()];
     int i = 0;
     for(DatanodeStorageInfo s : blocksMap.getStorages(block)) {
@@ -3409,7 +3409,7 @@ public class BlockManager {
     }
   }
 
-  public BlockInfoContiguous getStoredBlock(Block block) {
+  public BlockInfo getStoredBlock(Block block) {
     return blocksMap.getStoredBlock(block);
   }
 
@@ -3462,8 +3462,8 @@ public class BlockManager {
    * replicated.
    */
   public boolean checkBlocksProperlyReplicated(
-      String src, BlockInfoContiguous[] blocks) {
-    for (BlockInfoContiguous b: blocks) {
+      String src, BlockInfo[] blocks) {
+    for (BlockInfo b: blocks) {
       if (!b.isComplete()) {
         final BlockInfoContiguousUnderConstruction uc =
             (BlockInfoContiguousUnderConstruction)b;
@@ -3578,7 +3578,7 @@ public class BlockManager {
     return this.neededReplications.getCorruptReplOneBlockSize();
   }
 
-  public BlockInfoContiguous addBlockCollection(BlockInfoContiguous block,
+  public BlockInfo addBlockCollection(BlockInfo block,
       BlockCollection bc) {
     return blocksMap.addBlockCollection(block, bc);
   }
@@ -3818,7 +3818,7 @@ public class BlockManager {
 
   /**
    * A simple result enum for the result of
-   * {@link BlockManager#processMisReplicatedBlock(BlockInfoContiguous)}.
+   * {@link BlockManager#processMisReplicatedBlock(BlockInfo)}.
    */
   enum MisReplicationResult {
     /** The block should be invalidated since it belongs to a deleted file. */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index 5e7d34f..d06d503 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -36,10 +36,10 @@ import com.google.common.collect.Iterables;
  */
 class BlocksMap {
   private static class StorageIterator implements Iterator<DatanodeStorageInfo> {
-    private final BlockInfoContiguous blockInfo;
+    private final BlockInfo blockInfo;
     private int nextIdx = 0;
       
-    StorageIterator(BlockInfoContiguous blkInfo) {
+    StorageIterator(BlockInfo blkInfo) {
       this.blockInfo = blkInfo;
     }
 
@@ -63,14 +63,14 @@ class BlocksMap {
   /** Constant {@link LightWeightGSet} capacity. */
   private final int capacity;
   
-  private GSet<Block, BlockInfoContiguous> blocks;
+  private GSet<Block, BlockInfo> blocks;
 
   BlocksMap(int capacity) {
     // Use 2% of total memory to size the GSet capacity
     this.capacity = capacity;
-    this.blocks = new LightWeightGSet<Block, BlockInfoContiguous>(capacity) {
+    this.blocks = new LightWeightGSet<Block, BlockInfo>(capacity) {
       @Override
-      public Iterator<BlockInfoContiguous> iterator() {
+      public Iterator<BlockInfo> iterator() {
         SetIterator iterator = new SetIterator();
         /*
          * Not tracking any modifications to set. As this set will be used
@@ -97,15 +97,15 @@ class BlocksMap {
   }
 
   BlockCollection getBlockCollection(Block b) {
-    BlockInfoContiguous info = blocks.get(b);
+    BlockInfo info = blocks.get(b);
     return (info != null) ? info.getBlockCollection() : null;
   }
 
   /**
    * Add block b belonging to the specified block collection to the map.
    */
-  BlockInfoContiguous addBlockCollection(BlockInfoContiguous b, BlockCollection bc) {
-    BlockInfoContiguous info = blocks.get(b);
+  BlockInfo addBlockCollection(BlockInfo b, BlockCollection bc) {
+    BlockInfo info = blocks.get(b);
     if (info != b) {
       info = b;
       blocks.put(info);
@@ -120,7 +120,7 @@ class BlocksMap {
    * and remove all data-node locations associated with the block.
    */
   void removeBlock(Block block) {
-    BlockInfoContiguous blockInfo = blocks.remove(block);
+    BlockInfo blockInfo = blocks.remove(block);
     if (blockInfo == null)
       return;
 
@@ -132,7 +132,7 @@ class BlocksMap {
   }
   
   /** Returns the block object it it exists in the map. */
-  BlockInfoContiguous getStoredBlock(Block b) {
+  BlockInfo getStoredBlock(Block b) {
     return blocks.get(b);
   }
 
@@ -164,7 +164,7 @@ class BlocksMap {
    * For a block that has already been retrieved from the BlocksMap
    * returns {@link Iterable} of the storages the block belongs to.
    */
-  Iterable<DatanodeStorageInfo> getStorages(final BlockInfoContiguous storedBlock) {
+  Iterable<DatanodeStorageInfo> getStorages(final BlockInfo storedBlock) {
     return new Iterable<DatanodeStorageInfo>() {
       @Override
       public Iterator<DatanodeStorageInfo> iterator() {
@@ -175,7 +175,7 @@ class BlocksMap {
 
   /** counts number of containing nodes. Better than using iterator. */
   int numNodes(Block b) {
-    BlockInfoContiguous info = blocks.get(b);
+    BlockInfo info = blocks.get(b);
     return info == null ? 0 : info.numNodes();
   }
 
@@ -185,7 +185,7 @@ class BlocksMap {
    * only if it does not belong to any file and data-nodes.
    */
   boolean removeNode(Block b, DatanodeDescriptor node) {
-    BlockInfoContiguous info = blocks.get(b);
+    BlockInfo info = blocks.get(b);
     if (info == null)
       return false;
 
@@ -203,7 +203,7 @@ class BlocksMap {
     return blocks.size();
   }
 
-  Iterable<BlockInfoContiguous> getBlocks() {
+  Iterable<BlockInfo> getBlocks() {
     return blocks;
   }
   
@@ -218,8 +218,8 @@ class BlocksMap {
    * @param newBlock - block for replacement
    * @return new block
    */
-  BlockInfoContiguous replaceBlock(BlockInfoContiguous newBlock) {
-    BlockInfoContiguous currentBlock = blocks.get(newBlock);
+  BlockInfo replaceBlock(BlockInfo newBlock) {
+    BlockInfo currentBlock = blocks.get(newBlock);
     assert currentBlock != null : "the block if not in blocksMap";
     // replace block in data-node lists
     for (int i = currentBlock.numNodes() - 1; i >= 0; i--) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
index bf5ece9..a0f3503 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
@@ -369,7 +369,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
    * @param file The file.
    */
   private void rescanFile(CacheDirective directive, INodeFile file) {
-    BlockInfoContiguous[] blockInfos = file.getBlocks();
+    BlockInfo[] blockInfos = file.getBlocks();
 
     // Increment the "needed" statistics
     directive.addFilesNeeded(1);
@@ -394,7 +394,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
     }
 
     long cachedTotal = 0;
-    for (BlockInfoContiguous blockInfo : blockInfos) {
+    for (BlockInfo blockInfo : blockInfos) {
       if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) {
         // We don't try to cache blocks that are under construction.
         LOG.trace("Directive {}: can't cache block {} because it is in state "
@@ -453,7 +453,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
   }
 
   private String findReasonForNotCaching(CachedBlock cblock, 
-          BlockInfoContiguous blockInfo) {
+          BlockInfo blockInfo) {
     if (blockInfo == null) {
       // Somehow, a cache report with the block arrived, but the block
       // reports from the DataNode haven't (yet?) described such a block.
@@ -513,7 +513,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
           iter.remove();
         }
       }
-      BlockInfoContiguous blockInfo = blockManager.
+      BlockInfo blockInfo = blockManager.
             getStoredBlock(new Block(cblock.getBlockId()));
       String reason = findReasonForNotCaching(cblock, blockInfo);
       int neededCached = 0;
@@ -628,7 +628,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
       List<DatanodeDescriptor> pendingCached) {
     // To figure out which replicas can be cached, we consult the
     // blocksMap.  We don't want to try to cache a corrupt replica, though.
-    BlockInfoContiguous blockInfo = blockManager.
+    BlockInfo blockInfo = blockManager.
           getStoredBlock(new Block(cachedBlock.getBlockId()));
     if (blockInfo == null) {
       LOG.debug("Block {}: can't add new cached replicas," +
@@ -667,7 +667,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
       Iterator<CachedBlock> it = datanode.getPendingCached().iterator();
       while (it.hasNext()) {
         CachedBlock cBlock = it.next();
-        BlockInfoContiguous info =
+        BlockInfo info =
             blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
         if (info != null) {
           pendingBytes -= info.getNumBytes();
@@ -677,7 +677,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
       // Add pending uncached blocks from effective capacity
       while (it.hasNext()) {
         CachedBlock cBlock = it.next();
-        BlockInfoContiguous info =
+        BlockInfo info =
             blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
         if (info != null) {
           pendingBytes += info.getNumBytes();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 4731ad4..02abc1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -335,7 +335,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
    * Remove block from the list of blocks belonging to the data-node. Remove
    * data-node from the block.
    */
-  boolean removeBlock(BlockInfoContiguous b) {
+  boolean removeBlock(BlockInfo b) {
     final DatanodeStorageInfo s = b.findStorageInfo(this);
     // if block exists on this datanode
     if (s != null) {
@@ -348,7 +348,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
    * Remove block from the list of blocks belonging to the data-node. Remove
    * data-node from the block.
    */
-  boolean removeBlock(String storageID, BlockInfoContiguous b) {
+  boolean removeBlock(String storageID, BlockInfo b) {
     DatanodeStorageInfo s = getStorageInfo(storageID);
     if (s != null) {
       return s.removeBlock(b);
@@ -537,12 +537,12 @@ public class DatanodeDescriptor extends DatanodeInfo {
     }
   }
 
-  private static class BlockIterator implements Iterator<BlockInfoContiguous> {
+  private static class BlockIterator implements Iterator<BlockInfo> {
     private int index = 0;
-    private final List<Iterator<BlockInfoContiguous>> iterators;
+    private final List<Iterator<BlockInfo>> iterators;
     
     private BlockIterator(final DatanodeStorageInfo... storages) {
-      List<Iterator<BlockInfoContiguous>> iterators = new ArrayList<Iterator<BlockInfoContiguous>>();
+      List<Iterator<BlockInfo>> iterators = new ArrayList<Iterator<BlockInfo>>();
       for (DatanodeStorageInfo e : storages) {
         iterators.add(e.getBlockIterator());
       }
@@ -556,7 +556,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
     }
 
     @Override
-    public BlockInfoContiguous next() {
+    public BlockInfo next() {
       update();
       return iterators.get(index).next();
     }
@@ -573,10 +573,10 @@ public class DatanodeDescriptor extends DatanodeInfo {
     }
   }
 
-  Iterator<BlockInfoContiguous> getBlockIterator() {
+  Iterator<BlockInfo> getBlockIterator() {
     return new BlockIterator(getStorageInfos());
   }
-  Iterator<BlockInfoContiguous> getBlockIterator(final String storageID) {
+  Iterator<BlockInfo> getBlockIterator(final String storageID) {
     return new BlockIterator(getStorageInfo(storageID));
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index c6c9001..57e81b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -25,9 +25,6 @@ import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
 import org.apache.hadoop.hdfs.server.protocol.StorageReport;
@@ -83,10 +80,10 @@ public class DatanodeStorageInfo {
   /**
    * Iterates over the list of blocks belonging to the data-node.
    */
-  class BlockIterator implements Iterator<BlockInfoContiguous> {
-    private BlockInfoContiguous current;
+  class BlockIterator implements Iterator<BlockInfo> {
+    private BlockInfo current;
 
-    BlockIterator(BlockInfoContiguous head) {
+    BlockIterator(BlockInfo head) {
       this.current = head;
     }
 
@@ -94,8 +91,8 @@ public class DatanodeStorageInfo {
       return current != null;
     }
 
-    public BlockInfoContiguous next() {
-      BlockInfoContiguous res = current;
+    public BlockInfo next() {
+      BlockInfo res = current;
       current = current.getNext(current.findStorageInfo(DatanodeStorageInfo.this));
       return res;
     }
@@ -115,7 +112,7 @@ public class DatanodeStorageInfo {
   private volatile long remaining;
   private long blockPoolUsed;
 
-  private volatile BlockInfoContiguous blockList = null;
+  private volatile BlockInfo blockList = null;
   private int numBlocks = 0;
 
   // The ID of the last full block report which updated this storage.
@@ -229,7 +226,7 @@ public class DatanodeStorageInfo {
     return blockPoolUsed;
   }
 
-  public AddBlockResult addBlock(BlockInfoContiguous b) {
+  public AddBlockResult addBlock(BlockInfo b) {
     // First check whether the block belongs to a different storage
     // on the same DN.
     AddBlockResult result = AddBlockResult.ADDED;
@@ -254,7 +251,7 @@ public class DatanodeStorageInfo {
     return result;
   }
 
-  public boolean removeBlock(BlockInfoContiguous b) {
+  public boolean removeBlock(BlockInfo b) {
     blockList = b.listRemove(blockList, this);
     if (b.removeStorage(this)) {
       numBlocks--;
@@ -268,7 +265,7 @@ public class DatanodeStorageInfo {
     return numBlocks;
   }
   
-  Iterator<BlockInfoContiguous> getBlockIterator() {
+  Iterator<BlockInfo> getBlockIterator() {
     return new BlockIterator(blockList);
 
   }
@@ -277,7 +274,7 @@ public class DatanodeStorageInfo {
    * Move block to the head of the list of blocks belonging to the data-node.
    * @return the index of the head of the blockList
    */
-  int moveBlockToHead(BlockInfoContiguous b, int curIndex, int headIndex) {
+  int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) {
     blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex);
     return curIndex;
   }
@@ -287,7 +284,7 @@ public class DatanodeStorageInfo {
    * @return the head of the blockList
    */
   @VisibleForTesting
-  BlockInfoContiguous getBlockListHeadForTesting(){
+  BlockInfo getBlockListHeadForTesting(){
     return blockList;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index 5f7366e..639d2b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -100,7 +100,7 @@ public class DecommissionManager {
    * reports or other events. Before being finally marking as decommissioned,
    * another check is done with the actual block map.
    */
-  private final TreeMap<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>
+  private final TreeMap<DatanodeDescriptor, AbstractList<BlockInfo>>
       decomNodeBlocks;
 
   /**
@@ -248,7 +248,7 @@ public class DecommissionManager {
    * Full-strength replication is not always necessary, hence "sufficient".
    * @return true if sufficient, else false.
    */
-  private boolean isSufficientlyReplicated(BlockInfoContiguous block, 
+  private boolean isSufficientlyReplicated(BlockInfo block,
       BlockCollection bc,
       NumberReplicas numberReplicas) {
     final int numExpected = bc.getPreferredBlockReplication();
@@ -412,7 +412,7 @@ public class DecommissionManager {
     }
 
     private void check() {
-      final Iterator<Map.Entry<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>>
+      final Iterator<Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>>>
           it = new CyclicIteration<>(decomNodeBlocks, iterkey).iterator();
       final LinkedList<DatanodeDescriptor> toRemove = new LinkedList<>();
 
@@ -420,10 +420,10 @@ public class DecommissionManager {
           && !exceededNumBlocksPerCheck()
           && !exceededNumNodesPerCheck()) {
         numNodesChecked++;
-        final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>
+        final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>>
             entry = it.next();
         final DatanodeDescriptor dn = entry.getKey();
-        AbstractList<BlockInfoContiguous> blocks = entry.getValue();
+        AbstractList<BlockInfo> blocks = entry.getValue();
         boolean fullScan = false;
         if (blocks == null) {
           // This is a newly added datanode, run through its list to schedule 
@@ -495,7 +495,7 @@ public class DecommissionManager {
      * datanode.
      */
     private void pruneSufficientlyReplicated(final DatanodeDescriptor datanode,
-        AbstractList<BlockInfoContiguous> blocks) {
+        AbstractList<BlockInfo> blocks) {
       processBlocksForDecomInternal(datanode, blocks.iterator(), null, true);
     }
 
@@ -509,9 +509,9 @@ public class DecommissionManager {
      * @param datanode
      * @return List of insufficiently replicated blocks 
      */
-    private AbstractList<BlockInfoContiguous> handleInsufficientlyReplicated(
+    private AbstractList<BlockInfo> handleInsufficientlyReplicated(
         final DatanodeDescriptor datanode) {
-      AbstractList<BlockInfoContiguous> insufficient = new ChunkedArrayList<>();
+      AbstractList<BlockInfo> insufficient = new ChunkedArrayList<>();
       processBlocksForDecomInternal(datanode, datanode.getBlockIterator(),
           insufficient, false);
       return insufficient;
@@ -535,8 +535,8 @@ public class DecommissionManager {
      */
     private void processBlocksForDecomInternal(
         final DatanodeDescriptor datanode,
-        final Iterator<BlockInfoContiguous> it,
-        final List<BlockInfoContiguous> insufficientlyReplicated,
+        final Iterator<BlockInfo> it,
+        final List<BlockInfo> insufficientlyReplicated,
         boolean pruneSufficientlyReplicated) {
       boolean firstReplicationLog = true;
       int underReplicatedBlocks = 0;
@@ -544,7 +544,7 @@ public class DecommissionManager {
       int underReplicatedInOpenFiles = 0;
       while (it.hasNext()) {
         numBlocksChecked++;
-        final BlockInfoContiguous block = it.next();
+        final BlockInfo block = it.next();
         // Remove the block from the list if it's no longer in the block map,
         // e.g. the containing file has been deleted
         if (blockManager.blocksMap.getStoredBlock(block) == null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 307bd59..3b5f973 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
@@ -235,7 +235,7 @@ class FSDirWriteFileOp {
         return onRetryBlock[0];
       } else {
         // add new chosen targets to already allocated block and return
-        BlockInfoContiguous lastBlockInFile = pendingFile.getLastBlock();
+        BlockInfo lastBlockInFile = pendingFile.getLastBlock();
         ((BlockInfoContiguousUnderConstruction) lastBlockInFile)
             .setExpectedLocations(targets);
         offset = pendingFile.computeFileSize();
@@ -507,7 +507,7 @@ class FSDirWriteFileOp {
   /**
    * Add a block to the file. Returns a reference to the added block.
    */
-  private static BlockInfoContiguous addBlock(
+  private static BlockInfo addBlock(
       FSDirectory fsd, String path, INodesInPath inodesInPath, Block block,
       DatanodeStorageInfo[] targets) throws IOException {
     fsd.writeLock();
@@ -608,7 +608,7 @@ class FSDirWriteFileOp {
       }
     }
     final INodeFile file = fsn.checkLease(src, clientName, inode, fileId);
-    BlockInfoContiguous lastBlockInFile = file.getLastBlock();
+    BlockInfo lastBlockInFile = file.getLastBlock();
     if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
       // The block that the client claims is the current last block
       // doesn't match up with what we think is the last block. There are
@@ -636,7 +636,7 @@ class FSDirWriteFileOp {
       //    changed the namesystem state yet.
       //    We run this analysis again in Part II where case 4 is impossible.
 
-      BlockInfoContiguous penultimateBlock = file.getPenultimateBlock();
+      BlockInfo penultimateBlock = file.getPenultimateBlock();
       if (previous == null &&
           lastBlockInFile != null &&
           lastBlockInFile.getNumBytes() >= file.getPreferredBlockSize() &&
@@ -762,7 +762,7 @@ class FSDirWriteFileOp {
       long id, PermissionStatus permissions, long mtime, long atime,
       short replication, long preferredBlockSize, byte storagePolicyId) {
     return new INodeFile(id, null, permissions, mtime, atime,
-        BlockInfoContiguous.EMPTY_ARRAY, replication, preferredBlockSize,
+        BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize,
         storagePolicyId);
   }
 
@@ -802,7 +802,7 @@ class FSDirWriteFileOp {
       DatanodeStorageInfo[] targets)
       throws IOException {
     assert fsn.hasWriteLock();
-    BlockInfoContiguous b = addBlock(fsn.dir, src, inodesInPath, newBlock,
+    BlockInfo b = addBlock(fsn.dir, src, inodesInPath, newBlock,
                                      targets);
     NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
     DatanodeStorageInfo.incrementBlocksScheduled(targets);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 8fdd2d7..5b8650e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -33,10 +33,8 @@ import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.fs.UnresolvedLinkException;
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.XAttrHelper;
@@ -51,7 +49,7 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -919,7 +917,7 @@ public class FSDirectory implements Closeable {
         unprotectedTruncate(iip, newLength, collectedBlocks, mtime, null);
 
     if(! onBlockBoundary) {
-      BlockInfoContiguous oldBlock = file.getLastBlock();
+      BlockInfo oldBlock = file.getLastBlock();
       Block tBlk =
       getFSNamesystem().prepareFileForTruncate(iip,
           clientName, clientMachine, file.computeFileSize() - newLength,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 83e52bc..dec9709 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.Storage.FormatConfirmable;
@@ -773,10 +773,10 @@ public class FSEditLog implements LogsPurgeable {
   
   public void logAddBlock(String path, INodeFile file) {
     Preconditions.checkArgument(file.isUnderConstruction());
-    BlockInfoContiguous[] blocks = file.getBlocks();
+    BlockInfo[] blocks = file.getBlocks();
     Preconditions.checkState(blocks != null && blocks.length > 0);
-    BlockInfoContiguous pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null;
-    BlockInfoContiguous lastBlock = blocks[blocks.length - 1];
+    BlockInfo pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null;
+    BlockInfo lastBlock = blocks[blocks.length - 1];
     AddBlockOp op = AddBlockOp.getInstance(cache.get()).setPath(path)
         .setPenultimateBlock(pBlock).setLastBlock(lastBlock);
     logEdit(op);


[02/50] [abbrv] hadoop git commit: HADOOP-9891. CLIMiniCluster instructions fail with MiniYarnCluster ClassNotFoundException (Darrell Taylor via aw)

Posted by zj...@apache.org.
HADOOP-9891. CLIMiniCluster instructions fail with MiniYarnCluster ClassNotFoundException (Darrell Taylor via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cd01a19
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cd01a19
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cd01a19

Branch: refs/heads/YARN-2928
Commit: 0cd01a198b8885b63a635bf84b2efc6373b39fa3
Parents: 8b88df7
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed May 27 16:36:55 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 15:46:59 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt                   | 3 +++
 .../hadoop-common/src/site/markdown/CLIMiniCluster.md.vm          | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd01a19/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 552443c..d29a885 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -463,6 +463,9 @@ Trunk (Unreleased)
     HADOOP-11775. Fix Javadoc typos in hadoop-openstack module (Yanjun Wang
     via aw)
 
+    HADOOP-9891. CLIMiniCluster instructions fail with MiniYarnCluster
+    ClassNotFoundException (Darrell Taylor via aw)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd01a19/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
index 25fecda..c7da62b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
@@ -44,7 +44,7 @@ Running the MiniCluster
 
 From inside the root directory of the extracted tarball, you can start the CLI MiniCluster using the following command:
 
-    $ bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${project.version}-tests.jar minicluster -rmport RM_PORT -jhsport JHS_PORT
+    $ HADOOP_CLASSPATH=share/hadoop/yarn/test/hadoop-yarn-server-tests-${project.version}-tests.jar bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${project.version}-tests.jar minicluster -rmport RM_PORT -jhsport JHS_PORT
 
 In the example command above, `RM_PORT` and `JHS_PORT` should be replaced by the user's choice of these port numbers. If not specified, random free ports will be used.
 


[11/50] [abbrv] hadoop git commit: HADOOP-11406. xargs -P is not portable (Kengo Seki via aw)

Posted by zj...@apache.org.
HADOOP-11406. xargs -P is not portable (Kengo Seki via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c65e8ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c65e8ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c65e8ab

Branch: refs/heads/YARN-2928
Commit: 6c65e8ab54c51f234dc4b269cc6decd754c1f86b
Parents: de39fed
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 10:36:40 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  2 +
 .../src/main/bin/hadoop-functions.sh            | 42 +++++++++++---------
 .../main/conf/hadoop-user-functions.sh.example  | 29 +++++++++++++-
 3 files changed, 54 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c65e8ab/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index eb1db29..7da02ed 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -466,6 +466,8 @@ Trunk (Unreleased)
     HADOOP-9891. CLIMiniCluster instructions fail with MiniYarnCluster
     ClassNotFoundException (Darrell Taylor via aw)
 
+    HADOOP-11406. xargs -P is not portable (Kengo Seki via aw)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c65e8ab/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 67e8870..5556f2f 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -461,27 +461,33 @@ function hadoop_connect_to_hosts
     if [[ -z "${SLAVE_NAMES}" ]]; then
       SLAVE_NAMES=$(sed 's/#.*$//;/^$/d' "${SLAVE_FILE}")
     fi
-
-    # quoting here gets tricky. it's easier to push it into a function
-    # so that we don't have to deal with it. However...
-    # xargs can't use a function so instead we'll export it out
-    # and force it into a subshell
-    # moral of the story: just use pdsh.
-    export -f hadoop_actual_ssh
-    export HADOOP_SSH_OPTS
-
-    # xargs is used with option -I to replace the placeholder in arguments
-    # list with each hostname read from stdin/pipe. But it consider one
-    # line as one argument while reading from stdin/pipe. So place each
-    # hostname in different lines while passing via pipe.
-    SLAVE_NAMES=$(echo "$SLAVE_NAMES" | tr ' ' '\n' )
-    echo "${SLAVE_NAMES}" | \
-    xargs -n 1 -P"${HADOOP_SSH_PARALLEL}" \
-    -I {} bash -c --  "hadoop_actual_ssh {} ${params}"
-    wait
+    hadoop_connect_to_hosts_without_pdsh "${params}"
   fi
 }
 
+## @description  Connect to ${SLAVE_NAMES} and execute command
+## @description  under the environment which does not support pdsh.
+## @audience     private
+## @stability    evolving
+## @replaceable  yes
+## @param        command
+## @param        [...]
+function hadoop_connect_to_hosts_without_pdsh
+{
+  # shellcheck disable=SC2124
+  local params="$@"
+  local slaves=(${SLAVE_NAMES})
+  for (( i = 0; i < ${#slaves[@]}; i++ ))
+  do
+    if (( i != 0 && i % HADOOP_SSH_PARALLEL == 0 )); then
+      wait
+    fi
+    # shellcheck disable=SC2086
+    hadoop_actual_ssh "${slaves[$i]}" ${params} &
+  done
+  wait
+}
+
 ## @description  Utility routine to handle --slaves mode
 ## @audience     private
 ## @stability    evolving

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c65e8ab/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
index b2f78f8..3cf5776 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
@@ -50,7 +50,7 @@
 #
 
 #
-# Another example:  finding java
+# Example:  finding java
 #
 # By default, Hadoop assumes that $JAVA_HOME is always defined
 # outside of its configuration. Eons ago, Apple standardized
@@ -85,3 +85,30 @@
 #    exit 1
 #  fi
 #}
+
+#
+# Example:  efficient command execution for the slaves
+#
+# To improve performance, you can use xargs -P
+# instead of the for loop, if supported.
+#
+#function hadoop_connect_to_hosts_without_pdsh
+#{
+#  # quoting here gets tricky. it's easier to push it into a function
+#  # so that we don't have to deal with it. However...
+#  # xargs can't use a function so instead we'll export it out
+#  # and force it into a subshell
+#  # moral of the story: just use pdsh.
+#  export -f hadoop_actual_ssh
+#  export HADOOP_SSH_OPTS
+#
+#  # xargs is used with option -I to replace the placeholder in arguments
+#  # list with each hostname read from stdin/pipe. But it consider one
+#  # line as one argument while reading from stdin/pipe. So place each
+#  # hostname in different lines while passing via pipe.
+#  SLAVE_NAMES=$(echo "$SLAVE_NAMES" | tr ' ' '\n' )
+#  echo "${SLAVE_NAMES}" | \
+#  xargs -n 1 -P"${HADOOP_SSH_PARALLEL}" \
+#  -I {} bash -c --  "hadoop_actual_ssh {} ${params}"
+#  wait
+#}


[39/50] [abbrv] hadoop git commit: YARN-2900. Application (Attempt and Container) Not Found in AHS results in Internal Server Error (500). Contributed by Zhijie Shen and Mit Desai

Posted by zj...@apache.org.
YARN-2900. Application (Attempt and Container) Not Found in AHS results
in Internal Server Error (500). Contributed by Zhijie Shen and Mit Desai


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0ecae35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0ecae35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0ecae35

Branch: refs/heads/YARN-2928
Commit: c0ecae35275a96d69c5e5148f0fec91866a14950
Parents: e2101ca
Author: Xuan <xg...@apache.org>
Authored: Sat May 30 21:56:19 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:59 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../timeline/webapp/TimelineWebServices.java    |  2 -
 .../TestApplicationHistoryClientService.java    | 70 +++++++++++++++++++-
 .../hadoop/yarn/server/webapp/WebServices.java  | 27 +++++---
 4 files changed, 89 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ecae35/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6f5faa9..45c4f68 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -694,6 +694,9 @@ Release 2.7.1 - UNRELEASED
     YARN-3686. CapacityScheduler should trim default_node_label_expression. 
     (Sunil G via wangda)
 
+    YARN-2900. Application (Attempt and Container) Not Found in AHS results
+    in InternalServer Error (500). (Zhijie Shen and Mit Desai via xgong)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ecae35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
index 90f4d39..40e9fed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
@@ -47,7 +47,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@@ -61,7 +60,6 @@ import org.apache.hadoop.yarn.server.timeline.NameValuePair;
 import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
 import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
-import org.apache.hadoop.yarn.util.YarnVersionInfo;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 import org.apache.hadoop.yarn.webapp.ForbiddenException;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ecae35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 764ecb9..d04903c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
@@ -54,12 +57,13 @@ public class TestApplicationHistoryClientService {
 
   private static ApplicationHistoryClientService clientService;
   private static TimelineDataManager dataManager;
+  private final static int MAX_APPS = 2;
 
   @BeforeClass
   public static void setup() throws Exception {
     Configuration conf = new YarnConfiguration();
     TimelineStore store =
-        TestApplicationHistoryManagerOnTimelineStore.createStore(2);
+        TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
     TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
     dataManager =
         new TimelineDataManager(store, aclsManager);
@@ -72,6 +76,70 @@ public class TestApplicationHistoryClientService {
   }
 
   @Test
+  public void testApplicationNotFound() throws IOException, YarnException {
+    ApplicationId appId = null;
+    appId = ApplicationId.newInstance(0, MAX_APPS + 1);
+    GetApplicationReportRequest request =
+        GetApplicationReportRequest.newInstance(appId);
+    try {
+      @SuppressWarnings("unused")
+      GetApplicationReportResponse response =
+          clientService.getApplicationReport(request);
+      Assert.fail("Exception should have been thrown before we reach here.");
+    } catch (ApplicationNotFoundException e) {
+      //This exception is expected.
+      Assert.assertTrue(e.getMessage().contains(
+          "doesn't exist in the timeline store"));
+    } catch (Exception e) {
+      Assert.fail("Undesired exception caught");
+    }
+  }
+
+  @Test
+  public void testApplicationAttemptNotFound() throws IOException, YarnException {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
+    GetApplicationAttemptReportRequest request =
+        GetApplicationAttemptReportRequest.newInstance(appAttemptId);
+    try {
+      @SuppressWarnings("unused")
+      GetApplicationAttemptReportResponse response =
+          clientService.getApplicationAttemptReport(request);
+      Assert.fail("Exception should have been thrown before we reach here.");
+    } catch (ApplicationAttemptNotFoundException e) {
+      //This Exception is expected
+      System.out.println(e.getMessage());
+      Assert.assertTrue(e.getMessage().contains(
+          "doesn't exist in the timeline store"));
+    } catch (Exception e) {
+      Assert.fail("Undesired exception caught");
+    }
+  }
+
+  @Test
+  public void testContainerNotFound() throws IOException, YarnException {
+   ApplicationId appId = ApplicationId.newInstance(0, 1);
+   ApplicationAttemptId appAttemptId =
+       ApplicationAttemptId.newInstance(appId, 1);
+   ContainerId containerId = ContainerId.newContainerId(appAttemptId,
+       MAX_APPS + 1);
+   GetContainerReportRequest request =
+       GetContainerReportRequest.newInstance(containerId);
+   try {
+   @SuppressWarnings("unused")
+   GetContainerReportResponse response =
+       clientService.getContainerReport(request);
+   } catch (ContainerNotFoundException e) {
+     //This exception is expected
+     Assert.assertTrue(e.getMessage().contains(
+         "doesn't exist in the timeline store"));
+   }  catch (Exception e) {
+      Assert.fail("Undesired exception caught");
+   }
+ }
+
+  @Test
   public void testApplicationReport() throws IOException, YarnException {
     ApplicationId appId = null;
     appId = ApplicationId.newInstance(0, 1);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ecae35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index 8404719..ae27d45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -47,6 +47,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@@ -477,17 +480,21 @@ public class WebServices {
 
   private static void rewrapAndThrowException(Exception e) {
     if (e instanceof UndeclaredThrowableException) {
-      if (e.getCause() instanceof AuthorizationException) {
-        throw new ForbiddenException(e.getCause());
-      } else {
-        throw new WebApplicationException(e.getCause());
-      }
+      rewrapAndThrowThrowable(e.getCause());
     } else {
-      if (e instanceof AuthorizationException) {
-        throw new ForbiddenException(e);
-      } else {
-        throw new WebApplicationException(e);
-      }
+      rewrapAndThrowThrowable(e);
+    }
+  }
+
+  private static void rewrapAndThrowThrowable(Throwable t) {
+    if (t instanceof AuthorizationException) {
+      throw new ForbiddenException(t);
+    } if (t instanceof ApplicationNotFoundException ||
+        t instanceof ApplicationAttemptNotFoundException ||
+        t instanceof ContainerNotFoundException) {
+      throw new NotFoundException(t);
+    } else {
+      throw new WebApplicationException(t);
     }
   }
 


[38/50] [abbrv] hadoop git commit: YARN-3467. Expose allocatedMB, allocatedVCores, and runningContainers metrics on running Applications in RM Web UI. (Anubhav Dhoot via kasha)

Posted by zj...@apache.org.
YARN-3467. Expose allocatedMB, allocatedVCores, and runningContainers metrics on running Applications in RM Web UI. (Anubhav Dhoot via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af69d9c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af69d9c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af69d9c1

Branch: refs/heads/YARN-2928
Commit: af69d9c14484d9ece37d2ed505ea23ec63a891f9
Parents: d0c491b
Author: Karthik Kambatla <ka...@apache.org>
Authored: Sat May 30 13:36:24 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:59 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                       |  3 +++
 .../hadoop/yarn/server/webapp/WebPageUtils.java       |  4 ++--
 .../apache/hadoop/yarn/server/webapp/dao/AppInfo.java | 14 ++++++++++++++
 .../webapp/FairSchedulerAppsBlock.java                |  6 ++++++
 .../server/resourcemanager/webapp/RMAppsBlock.java    |  8 ++++++++
 5 files changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 339203c..2c48701 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -371,6 +371,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3713. Remove duplicate function call storeContainerDiagnostics in
     ContainerDiagnosticsUpdateTransition (zxu via rkanter)
 
+    YARN-3467. Expose allocatedMB, allocatedVCores, and runningContainers metrics on 
+    running Applications in RM Web UI. (Anubhav Dhoot via kasha)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
index 6ca5011..ed0fe38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
@@ -52,9 +52,9 @@ public class WebPageUtils {
       .append(", 'mRender': renderHadoopDate }")
       .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':");
     if (isFairSchedulerPage) {
-      sb.append("[11]");
+      sb.append("[13]");
     } else if (isResourceManager) {
-      sb.append("[10]");
+      sb.append("[12]");
     } else {
       sb.append("[9]");
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index 5d9ba6f..0cc5f75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -58,6 +58,8 @@ public class AppInfo {
   protected long finishedTime;
   protected long elapsedTime;
   protected String applicationTags;
+  private int allocatedCpuVcores;
+  private int allocatedMemoryMB;
 
   public AppInfo() {
     // JAXB needs this
@@ -86,6 +88,10 @@ public class AppInfo {
     if (app.getApplicationResourceUsageReport() != null) {
       runningContainers =
           app.getApplicationResourceUsageReport().getNumUsedContainers();
+      allocatedCpuVcores = app.getApplicationResourceUsageReport()
+          .getUsedResources().getVirtualCores();
+      allocatedMemoryMB = app.getApplicationResourceUsageReport()
+          .getUsedResources().getMemory();
     }
     progress = app.getProgress() * 100; // in percent
     if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
@@ -133,6 +139,14 @@ public class AppInfo {
     return runningContainers;
   }
 
+  public int getAllocatedCpuVcores() {
+    return allocatedCpuVcores;
+  }
+
+  public int getAllocatedMemoryMB() {
+    return allocatedMemoryMB;
+  }
+
   public float getProgress() {
     return progress;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index 80099d0..f257656 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -93,6 +93,8 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
             th(".state", "State").
             th(".finalstatus", "FinalStatus").
             th(".runningcontainer", "Running Containers").
+            th(".allocatedCpu", "Allocated CPU VCores").
+            th(".allocatedMemory", "Allocated Memory MB").
             th(".progress", "Progress").
             th(".ui", "Tracking UI")._()._().
         tbody();
@@ -136,6 +138,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
       .append(appInfo.getFinalStatus()).append("\",\"")
       .append(appInfo.getRunningContainers() == -1 ? "N/A" : String
          .valueOf(appInfo.getRunningContainers())).append("\",\"")
+      .append(appInfo.getAllocatedVCores() == -1 ? "N/A" : String
+        .valueOf(appInfo.getAllocatedVCores())).append("\",\"")
+      .append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
+        .valueOf(appInfo.getAllocatedMB())).append("\",\"")
       // Progress bar
       .append("<br title='").append(percent)
       .append("'> <div class='").append(C_PROGRESSBAR).append("' title='")

http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index 1d97bef..d252c30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -60,6 +60,8 @@ public class RMAppsBlock extends AppsBlock {
           .th(".finishtime", "FinishTime").th(".state", "State")
           .th(".finalstatus", "FinalStatus")
           .th(".runningcontainer", "Running Containers")
+          .th(".allocatedCpu", "Allocated CPU VCores")
+          .th(".allocatedMemory", "Allocated Memory MB")
           .th(".progress", "Progress")
           .th(".ui", "Tracking UI").th(".blacklisted", "Blacklisted Nodes")._()
           ._().tbody();
@@ -114,6 +116,12 @@ public class RMAppsBlock extends AppsBlock {
         .append(app.getRunningContainers() == -1 ? "N/A" : String
             .valueOf(app.getRunningContainers()))
         .append("\",\"")
+        .append(app.getAllocatedCpuVcores() == -1 ? "N/A" : String
+            .valueOf(app.getAllocatedCpuVcores()))
+        .append("\",\"")
+        .append(app.getAllocatedMemoryMB() == -1 ? "N/A" : String
+            .valueOf(app.getAllocatedMemoryMB()))
+        .append("\",\"")
         // Progress bar
         .append("<br title='").append(percent).append("'> <div class='")
         .append(C_PROGRESSBAR).append("' title='").append(join(percent, '%'))


[27/50] [abbrv] hadoop git commit: HADOOP-12022. fix site -Pdocs -Pdist in hadoop-project-dist; cleanout remaining forrest bits (aw)

Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
deleted file mode 100644
index 9880272..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
+++ /dev/null
@@ -1,29099 +0,0 @@
-<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<title>Hadoop  2.6.0 Release Notes</title>
-<STYLE type="text/css">
-	H1 {font-family: sans-serif}
-	H2 {font-family: sans-serif; margin-left: 7mm}
-	TABLE {margin-left: 7mm}
-</STYLE>
-</head>
-<body>
-<h1>Hadoop  2.6.0 Release Notes</h1>
-These release notes include new developer and user-facing incompatibilities, features, and major improvements. 
-<a name="changes"/>
-<h2>Changes since Hadoop 2.5.1</h2>
-<ul>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2853">YARN-2853</a>.
-     Major bug reported by Jian He and fixed by Jian He <br>
-     <b>Killing app may hang while AM is unregistering</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2846">YARN-2846</a>.
-     Blocker bug reported by Junping Du and fixed by Junping Du (nodemanager)<br>
-     <b>Incorrect persist exit code for running containers in reacquireContainer() that interrupted by NodeManager restart.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2843">YARN-2843</a>.
-     Major sub-task reported by Sushmitha Sreenivasan and fixed by Wangda Tan (resourcemanager)<br>
-     <b>NodeLabels manager should trim all inputs for hosts and labels</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2841">YARN-2841</a>.
-     Critical sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
-     <b>RMProxy should retry EOFException </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2834">YARN-2834</a>.
-     Blocker bug reported by Yesha Vora and fixed by Jian He <br>
-     <b>Resource manager crashed with Null Pointer Exception</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2830">YARN-2830</a>.
-     Blocker bug reported by Jonathan Eagles and fixed by Jonathan Eagles <br>
-     <b>Add backwords compatible ContainerId.newInstance constructor for use within Tez Local Mode</b><br>
-     <blockquote>I just committed this. Thanks [~jeagles] for the patch and [~ozawa] for the reviews!</blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2827">YARN-2827</a>.
-     Critical bug reported by Wangda Tan and fixed by Wangda Tan (client)<br>
-     <b>Fix bugs of yarn queue CLI</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2826">YARN-2826</a>.
-     Critical bug reported by Sidharta Seethana and fixed by Wangda Tan <br>
-     <b>User-Group mappings not updated by RM when a user is removed from a group.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2825">YARN-2825</a>.
-     Critical bug reported by Jian He and fixed by Jian He <br>
-     <b>Container leak on NM</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2824">YARN-2824</a>.
-     Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Capacity of labels should be zero by default</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2823">YARN-2823</a>.
-     Critical bug reported by Gour Saha and fixed by Jian He (resourcemanager)<br>
-     <b>NullPointerException in RM HA enabled 3-node cluster</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2819">YARN-2819</a>.
-     Critical bug reported by Gopal V and fixed by Zhijie Shen (timelineserver)<br>
-     <b>NPE in ATS Timeline Domains when upgrading from 2.4 to 2.6</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2818">YARN-2818</a>.
-     Critical bug reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Remove the logic to inject entity owner as the primary filter</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2813">YARN-2813</a>.
-     Major bug reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>NPE from MemoryTimelineStore.getDomains</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2812">YARN-2812</a>.
-     Major test reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>TestApplicationHistoryServer is likely to fail on less powerful machine</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2810">YARN-2810</a>.
-     Major test reported by Varun Vasudev and fixed by Varun Vasudev (resourcemanager)<br>
-     <b>TestRMProxyUsersConf fails on Windows VMs</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2805">YARN-2805</a>.
-     Blocker bug reported by Arpit Gupta and fixed by Wangda Tan (resourcemanager)<br>
-     <b>RM2 in HA setup tries to login using the RM1's kerberos principal</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2804">YARN-2804</a>.
-     Critical bug reported by Zhijie Shen and fixed by Zhijie Shen <br>
-     <b>Timeline server .out log have JAXB binding exceptions and warnings.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2803">YARN-2803</a>.
-     Critical bug reported by Chris Nauroth and fixed by Craig Welch (nodemanager)<br>
-     <b>MR distributed cache not working correctly on Windows after NodeManager privileged account changes.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2798">YARN-2798</a>.
-     Blocker bug reported by Arpit Gupta and fixed by Zhijie Shen (timelineserver)<br>
-     <b>YarnClient doesn't need to translate Kerberos name of timeline DT renewer</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2795">YARN-2795</a>.
-     Major sub-task reported by Phil D'Amore and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Resource Manager fails startup with HDFS label storage and secure cluster</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2794">YARN-2794</a>.
-     Major bug reported by Jian He and fixed by Jian He <br>
-     <b>Fix log msgs about distributing system-credentials </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2790">YARN-2790</a>.
-     Critical bug reported by Tassapol Athiapinya and fixed by Jian He (nodemanager)<br>
-     <b>NM can't aggregate logs past HDFS delegation token expiry.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2789">YARN-2789</a>.
-     Critical task reported by Siddharth Seth and fixed by Wangda Tan <br>
-     <b>Re-instate the NodeReport.newInstance API modified in YARN-2698</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2788">YARN-2788</a>.
-     Blocker bug reported by Gopal V and fixed by Xuan Gong (log-aggregation)<br>
-     <b>yarn logs -applicationId on 2.6.0 should support logs written by 2.4.0</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2785">YARN-2785</a>.
-     Major test reported by Varun Vasudev and fixed by Varun Vasudev <br>
-     <b>TestContainerResourceUsage fails intermittently</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2779">YARN-2779</a>.
-     Critical bug reported by Zhijie Shen and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
-     <b>SystemMetricsPublisher can use Kerberos directly instead of timeline DT</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2778">YARN-2778</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (client)<br>
-     <b>YARN node CLI should display labels on returned node reports</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2770">YARN-2770</a>.
-     Critical sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Timeline delegation tokens need to be automatically renewed by the RM</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2769">YARN-2769</a>.
-     Major bug reported by Varun Vasudev and fixed by Varun Vasudev (applications/distributed-shell)<br>
-     <b>Timeline server domain not set correctly when using shell_command on Windows</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2767">YARN-2767</a>.
-     Major test reported by Varun Vasudev and fixed by Varun Vasudev (resourcemanager)<br>
-     <b>RM web services - add test case to ensure the http static user cannot kill or submit apps in secure mode</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2760">YARN-2760</a>.
-     Trivial bug reported by Harsh J and fixed by Harsh J (documentation)<br>
-     <b>Completely remove word 'experimental' from FairScheduler docs</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2758">YARN-2758</a>.
-     Major test reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Update TestApplicationHistoryClientService to use the new generic history store</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2755">YARN-2755</a>.
-     Critical bug reported by Siqi Li and fixed by Siqi Li <br>
-     <b>NM fails to clean up usercache_DEL_&lt;timestamp&gt; dirs after YARN-661</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2753">YARN-2753</a>.
-     Major sub-task reported by zhihai xu and fixed by zhihai xu <br>
-     <b>Fix potential issues and code clean up for *NodeLabelsManager</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2752">YARN-2752</a>.
-     Critical bug reported by Xuan Gong and fixed by Xuan Gong <br>
-     <b>ContainerExecutor always append "nice -n" in command on branch-2</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2747">YARN-2747</a>.
-     Major test reported by Xuan Gong and fixed by Xuan Gong <br>
-     <b>TestAggregatedLogFormat fails in trunk</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2744">YARN-2744</a>.
-     Critical sub-task reported by Sumit Mohanty and fixed by Wangda Tan (capacityscheduler)<br>
-     <b>Under some scenario, it is possible to end up with capacity scheduler configuration that uses labels that no longer exist</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2743">YARN-2743</a>.
-     Blocker bug reported by Arpit Gupta and fixed by Jian He (resourcemanager)<br>
-     <b>Yarn jobs via oozie fail with failed to renew token (secure) or digest mismatch (unsecure) errors when RM is being killed</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2741">YARN-2741</a>.
-     Major bug reported by Craig Welch and fixed by Craig Welch (nodemanager)<br>
-     <b>Windows: Node manager cannot serve up log files via the web user interface when yarn.nodemanager.log-dirs to any drive letter other than C: (or, the drive that nodemanager is running on)</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2734">YARN-2734</a>.
-     Major bug reported by Sumit Mohanty and fixed by Xuan Gong (log-aggregation)<br>
-     <b>If a sub-folder is encountered by log aggregator it results in invalid aggregated file</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2732">YARN-2732</a>.
-     Major bug reported by Jian He and fixed by Jian He <br>
-     <b>Fix syntax error in SecureContainer.apt.vm</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2730">YARN-2730</a>.
-     Critical bug reported by Siqi Li and fixed by Siqi Li <br>
-     <b>DefaultContainerExecutor runs only one localizer at a time</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2726">YARN-2726</a>.
-     Minor sub-task reported by Phil D'Amore and fixed by Wangda Tan (capacityscheduler)<br>
-     <b>CapacityScheduler should explicitly log when an accessible label has no capacity</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2724">YARN-2724</a>.
-     Major bug reported by Sumit Mohanty and fixed by Xuan Gong (log-aggregation)<br>
-     <b>If an unreadable file is encountered during log aggregation then aggregated file in HDFS badly formed</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2723">YARN-2723</a>.
-     Major sub-task reported by Phil D'Amore and fixed by Naganarasimha G R (client)<br>
-     <b>rmadmin -replaceLabelsOnNode does not correctly parse port</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2721">YARN-2721</a>.
-     Blocker bug reported by Jian He and fixed by Jian He <br>
-     <b>Race condition: ZKRMStateStore retry logic may throw NodeExist exception </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2720">YARN-2720</a>.
-     Major bug reported by Craig Welch and fixed by Craig Welch (nodemanager)<br>
-     <b>Windows: Wildcard classpath variables not expanded against resources contained in archives</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2717">YARN-2717</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (log-aggregation)<br>
-     <b>containerLogNotFound log shows multiple time for the same container</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2715">YARN-2715</a>.
-     Blocker bug reported by Zhijie Shen and fixed by Zhijie Shen (resourcemanager)<br>
-     <b>Proxy user is problem for RPC interface if yarn.resourcemanager.webapp.proxyuser is not set.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2711">YARN-2711</a>.
-     Major test reported by Varun Vasudev and fixed by Varun Vasudev <br>
-     <b>TestDefaultContainerExecutor#testContainerLaunchError fails on Windows</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2709">YARN-2709</a>.
-     Major sub-task reported by Li Lu and fixed by Li Lu <br>
-     <b>Add retry for timeline client getDelegationToken method</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2707">YARN-2707</a>.
-     Minor bug reported by Ted Yu and fixed by Gera Shegalov <br>
-     <b>Potential null dereference in FSDownload</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2705">YARN-2705</a>.
-     Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
-     <b>Changes of RM node label manager default configuration</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2704">YARN-2704</a>.
-     Critical sub-task reported by Jian He and fixed by Jian He <br>
-     <b> Localization and log-aggregation will fail if hdfs delegation token expired after token-max-life-time</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2703">YARN-2703</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
-     <b>Add logUploadedTime into LogValue for better display</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2701">YARN-2701</a>.
-     Blocker bug reported by Xuan Gong and fixed by Xuan Gong <br>
-     <b>Potential race condition in startLocalizer when using LinuxContainerExecutor  </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2700">YARN-2700</a>.
-     Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api , resourcemanager)<br>
-     <b>TestSecureRMRegistryOperations failing on windows: auth problems</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2699">YARN-2699</a>.
-     Blocker sub-task reported by Wangda Tan and fixed by Wangda Tan (client)<br>
-     <b>Fix test timeout in TestResourceTrackerOnHA#testResourceTrackerOnHA</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2698">YARN-2698</a>.
-     Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
-     <b>Move getClusterNodeLabels and getNodeToLabels to YarnClient instead of AdminService</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2692">YARN-2692</a>.
-     Major sub-task reported by Steve Loughran and fixed by Steve Loughran <br>
-     <b>ktutil test hanging on some machines/ktutil versions</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2689">YARN-2689</a>.
-     Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api , resourcemanager)<br>
-     <b>TestSecureRMRegistryOperations failing on windows: secure ZK won't start</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2685">YARN-2685</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
-     <b>Resource on each label not correct when multiple NMs in a same host and some has label some not</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2682">YARN-2682</a>.
-     Minor bug reported by zhihai xu and fixed by zhihai xu (nodemanager)<br>
-     <b>WindowsSecureContainerExecutor should not depend on DefaultContainerExecutor#getFirstApplicationDir. </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2678">YARN-2678</a>.
-     Major sub-task reported by Gour Saha and fixed by Steve Loughran (api , resourcemanager)<br>
-     <b>Improved Yarn Registry service record structure</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2677">YARN-2677</a>.
-     Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api , resourcemanager)<br>
-     <b>registry punycoding of usernames doesn't fix all usernames to be DNS-valid</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2676">YARN-2676</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Timeline authentication filter should add support for proxy user</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2673">YARN-2673</a>.
-     Major sub-task reported by Li Lu and fixed by Li Lu <br>
-     <b>Add retry for timeline client put APIs</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2671">YARN-2671</a>.
-     Blocker bug reported by Zhijie Shen and fixed by Wangda Tan (resourcemanager)<br>
-     <b>ApplicationSubmissionContext change breaks the existing app submission</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2668">YARN-2668</a>.
-     Major sub-task reported by Steve Loughran and fixed by Steve Loughran (client)<br>
-     <b>yarn-registry JAR won't link against ZK 3.4.5</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2667">YARN-2667</a>.
-     Minor bug reported by Yi Liu and fixed by Yi Liu <br>
-     <b>Fix the release audit warning caused by hadoop-yarn-registry</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2662">YARN-2662</a>.
-     Minor bug reported by Chris Nauroth and fixed by Chris Nauroth (test)<br>
-     <b>TestCgroupsLCEResourcesHandler leaks file descriptors.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2656">YARN-2656</a>.
-     Major bug reported by Varun Vasudev and fixed by Zhijie Shen (resourcemanager)<br>
-     <b>RM web services authentication filter should add support for proxy user</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2652">YARN-2652</a>.
-     Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api)<br>
-     <b>add hadoop-yarn-registry package under hadoop-yarn</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2651">YARN-2651</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
-     <b>Spin off the LogRollingInterval from LogAggregationContext</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2649">YARN-2649</a>.
-     Major bug reported by Ming Ma and fixed by Ming Ma <br>
-     <b>Flaky test TestAMRMRPCNodeUpdates</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2647">YARN-2647</a>.
-     Major sub-task reported by Wangda Tan and fixed by Sunil G (client)<br>
-     <b>Add yarn queue CLI to get queue infos</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2644">YARN-2644</a>.
-     Major sub-task reported by Craig Welch and fixed by Craig Welch <br>
-     <b>Recalculate headroom more frequently to keep it accurate</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2635">YARN-2635</a>.
-     Major bug reported by Wei Yan and fixed by Wei Yan <br>
-     <b>TestRM, TestRMRestart, TestClientToAMTokens should run with both CS and FS</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2632">YARN-2632</a>.
-     Blocker sub-task reported by Junping Du and fixed by Junping Du (nodemanager)<br>
-     <b>Document NM Restart feature</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2630">YARN-2630</a>.
-     Major bug reported by Jian He and fixed by Jian He <br>
-     <b>TestDistributedShell#testDSRestartWithPreviousRunningContainers fails</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2629">YARN-2629</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Make distributed shell use the domain-based timeline ACLs</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2628">YARN-2628</a>.
-     Major bug reported by Varun Vasudev and fixed by Varun Vasudev (capacityscheduler)<br>
-     <b>Capacity scheduler with DominantResourceCalculator carries out reservation even though slots are free</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2627">YARN-2627</a>.
-     Major improvement reported by Xuan Gong and fixed by Xuan Gong <br>
-     <b>Add logs when attemptFailuresValidityInterval is enabled</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2624">YARN-2624</a>.
-     Blocker bug reported by Anubhav Dhoot and fixed by Anubhav Dhoot (nodemanager)<br>
-     <b>Resource Localization fails on a cluster due to existing cache directories</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2621">YARN-2621</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Simplify the output when the user doesn't have the access for getDomain(s) </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2617">YARN-2617</a>.
-     Major bug reported by Jun Gong and fixed by Jun Gong (nodemanager)<br>
-     <b>NM does not need to send finished container whose APP is not running to RM</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2615">YARN-2615</a>.
-     Blocker sub-task reported by Junping Du and fixed by Junping Du <br>
-     <b>ClientToAMTokenIdentifier and DelegationTokenIdentifier should allow extended fields</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2611">YARN-2611</a>.
-     Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler , resourcemanager , scheduler)<br>
-     <b>Fix jenkins findbugs warning and test case failures for trunk merge patch</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2610">YARN-2610</a>.
-     Major bug reported by Ray Chiang and fixed by Ray Chiang <br>
-     <b>Hamlet should close table tags</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2608">YARN-2608</a>.
-     Major bug reported by Wei Yan and fixed by Wei Yan <br>
-     <b>FairScheduler: Potential deadlocks in loading alloc files and clock access</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2607">YARN-2607</a>.
-     Major test reported by Ted Yu and fixed by Wangda Tan <br>
-     <b>TestDistributedShell fails in trunk</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2606">YARN-2606</a>.
-     Major bug reported by Mit Desai and fixed by Mit Desai (timelineserver)<br>
-     <b>Application History Server tries to access hdfs before doing secure login</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2602">YARN-2602</a>.
-     Major bug reported by Karam Singh and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Generic History Service of TimelineServer sometimes not able to handle NPE</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2596">YARN-2596</a>.
-     Major test reported by Junping Du and fixed by Karthik Kambatla <br>
-     <b>TestWorkPreservingRMRestart fails with FairScheduler</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2594">YARN-2594</a>.
-     Blocker bug reported by Karam Singh and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Potential deadlock in RM when querying ApplicationResourceUsageReport</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2591">YARN-2591</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>AHSWebServices should return FORBIDDEN(403) if the request user doesn't have access to the history data</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2588">YARN-2588</a>.
-     Major bug reported by Rohith and fixed by Rohith (resourcemanager)<br>
-     <b>Standby RM does not transitionToActive if previous transitionToActive is failed with ZK exception.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2584">YARN-2584</a>.
-     Major test reported by Zhijie Shen and fixed by Jian He <br>
-     <b>TestContainerManagerSecurity fails on trunk</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2583">YARN-2583</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
-     <b>Modify the LogDeletionService to support Log aggregation for LRS</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2582">YARN-2582</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
-     <b>Log related CLI and Web UI changes for Aggregated Logs in LRS</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2581">YARN-2581</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
-     <b>NMs need to find a way to get LogAggregationContext</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2579">YARN-2579</a>.
-     Blocker bug reported by Rohith and fixed by Rohith (resourcemanager)<br>
-     <b>Deadlock when EmbeddedElectorService and FatalEventDispatcher try to transition RM to StandBy at the same time</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2577">YARN-2577</a>.
-     Trivial improvement reported by Miklos Christine and fixed by Miklos Christine (documentation , fairscheduler)<br>
-     <b>Clarify ACL delimiter and how to configure ACL groups only</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2576">YARN-2576</a>.
-     Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler , resourcemanager , scheduler)<br>
-     <b>Prepare yarn-1051 branch for merging with trunk</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2569">YARN-2569</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
-     <b>API changes for handling logs of long-running services</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2568">YARN-2568</a>.
-     Major bug reported by Jian He and fixed by Jian He <br>
-     <b>TestAMRMClientOnRMRestart test fails</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2566">YARN-2566</a>.
-     Critical sub-task reported by zhihai xu and fixed by zhihai xu (nodemanager)<br>
-     <b>DefaultContainerExecutor should pick a working directory randomly</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2565">YARN-2565</a>.
-     Major bug reported by Karam Singh and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
-     <b>RM shouldn't use the old RMApplicationHistoryWriter unless explicitly setting FileSystemApplicationHistoryStore</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2563">YARN-2563</a>.
-     Blocker bug reported by Arpit Gupta and fixed by Zhijie Shen (timelineserver)<br>
-     <b>On secure clusters call to timeline server fails with authentication errors when running a job via oozie</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2562">YARN-2562</a>.
-     Critical bug reported by Vinod Kumar Vavilapalli and fixed by Tsuyoshi OZAWA <br>
-     <b>ContainerId@toString() is unreadable for epoch &gt;0 after YARN-2182</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2561">YARN-2561</a>.
-     Blocker sub-task reported by Tassapol Athiapinya and fixed by Junping Du <br>
-     <b>MR job client cannot reconnect to AM after NM restart.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2559">YARN-2559</a>.
-     Major bug reported by Karam Singh and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
-     <b>ResourceManager sometime become un-responsive due to NPE in SystemMetricsPublisher</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2558">YARN-2558</a>.
-     Blocker sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
-     <b>Updating ContainerTokenIdentifier#read/write to use ContainerId#getContainerId</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2557">YARN-2557</a>.
-     Major bug reported by Xuan Gong and fixed by Xuan Gong (applications/distributed-shell)<br>
-     <b>Add a parameter "attempt_Failures_Validity_Interval" in DistributedShell </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2549">YARN-2549</a>.
-     Minor test reported by Chris Nauroth and fixed by Chris Nauroth (nodemanager , test)<br>
-     <b>TestContainerLaunch fails due to classpath problem with hamcrest classes.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2547">YARN-2547</a>.
-     Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
-     <b>Cross Origin Filter throws UnsupportedOperationException upon destroy</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2546">YARN-2546</a>.
-     Major bug reported by Doug Haigh and fixed by Varun Vasudev (api)<br>
-     <b>REST API for application creation/submission is using strings for numeric &amp; boolean values</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2544">YARN-2544</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
-     <b>Common server side PB changes (not include user API PB changes)</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2542">YARN-2542</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
-     <b>"yarn application -status &lt;appId&gt;" throws NPE when retrieving the app from the timelineserver</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2541">YARN-2541</a>.
-     Major bug reported by Jian He and fixed by Jian He <br>
-     <b>Fix ResourceManagerRest.apt.vm syntax error</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2540">YARN-2540</a>.
-     Major bug reported by Ashwin Shankar and fixed by Ashwin Shankar (scheduler)<br>
-     <b>FairScheduler: Queue filters not working on scheduler page in RM UI</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2539">YARN-2539</a>.
-     Minor improvement reported by Wei Yan and fixed by Wei Yan <br>
-     <b>FairScheduler: Set the default value for maxAMShare to 0.5</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2538">YARN-2538</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
-     <b>Add logs when RM send new AMRMToken to ApplicationMaster</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2534">YARN-2534</a>.
-     Major bug reported by zhihai xu and fixed by zhihai xu (scheduler)<br>
-     <b>FairScheduler: Potential integer overflow calculating totalMaxShare</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2531">YARN-2531</a>.
-     Major improvement reported by Varun Vasudev and fixed by Varun Vasudev <br>
-     <b>CGroups - Admins should be allowed to enforce strict cpu limits</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2529">YARN-2529</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Generic history service RPC interface doesn't work when service authorization is enabled</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2528">YARN-2528</a>.
-     Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
-     <b>Cross Origin Filter Http response split vulnerability protection rejects valid origins</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2527">YARN-2527</a>.
-     Major bug reported by Benoy Antony and fixed by Benoy Antony (resourcemanager)<br>
-     <b>NPE in ApplicationACLsManager</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2526">YARN-2526</a>.
-     Critical bug reported by Wei Yan and fixed by Wei Yan (scheduler-load-simulator)<br>
-     <b>SLS can deadlock when all the threads are taken by AMSimulators</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2523">YARN-2523</a>.
-     Major bug reported by Nishan Shetty and fixed by Rohith (resourcemanager , webapp)<br>
-     <b>ResourceManager UI showing negative value for "Decommissioned Nodes" field</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2519">YARN-2519</a>.
-     Major test reported by Xiaoyu Yao and fixed by Xiaoyu Yao (webapp)<br>
-     <b>Credential Provider related unit tests failed on Windows</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2515">YARN-2515</a>.
-     Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
-     <b>Update ConverterUtils#toContainerId to parse epoch</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2512">YARN-2512</a>.
-     Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
-     <b>Allow for origin pattern matching in cross origin filter</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2511">YARN-2511</a>.
-     Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
-     <b>Allow All Origins by default when Cross Origin Filter is enabled</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2509">YARN-2509</a>.
-     Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
-     <b>Enable Cross Origin Filter for timeline server only and not all Yarn servers</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2508">YARN-2508</a>.
-     Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
-     <b>Cross Origin configuration parameters prefix are not honored</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2507">YARN-2507</a>.
-     Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (documentation , timelineserver)<br>
-     <b>Document Cross Origin Filter Configuration for ATS</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2505">YARN-2505</a>.
-     Major sub-task reported by Wangda Tan and fixed by Craig Welch (resourcemanager)<br>
-     <b>Support get/add/remove/change labels in RM REST API</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2504">YARN-2504</a>.
-     Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Support get/add/remove/change labels in RM admin CLI </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2503">YARN-2503</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Changes in RM Web UI to better show labels to end users</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2502">YARN-2502</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Changes in distributed shell to support specify labels</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2501">YARN-2501</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Changes in AMRMClient to support labels</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2500">YARN-2500</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Miscellaneous changes in ResourceManager to support labels</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2496">YARN-2496</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Changes for capacity scheduler to support allocate resource respect labels</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2494">YARN-2494</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
-     <b>Node label manager API and storage implementations</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2493">YARN-2493</a>.
-     Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api)<br>
-     <b>API changes for users</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2484">YARN-2484</a>.
-     Trivial bug reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
-     <b>FileSystemRMStateStore#readFile/writeFile should close FSData(In|Out)putStream in final block</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2475">YARN-2475</a>.
-     Major sub-task reported by Carlo Curino and fixed by Carlo Curino (resourcemanager)<br>
-     <b>ReservationSystem: replan upon capacity reduction</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2468">YARN-2468</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (log-aggregation , nodemanager , resourcemanager)<br>
-     <b>Log handling for LRS</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2462">YARN-2462</a>.
-     Major bug reported by Jason Lowe and fixed by Eric Payne <br>
-     <b>TestNodeManagerResync#testBlockNewContainerRequestsOnStartAndResync should have a test timeout</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2460">YARN-2460</a>.
-     Minor bug reported by Ray Chiang and fixed by Ray Chiang <br>
-     <b>Remove obsolete entries from yarn-default.xml</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2459">YARN-2459</a>.
-     Major bug reported by Mayank Bansal and fixed by Mayank Bansal (resourcemanager)<br>
-     <b>RM crashes if App gets rejected for any reason and HA is enabled</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2456">YARN-2456</a>.
-     Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
-     <b>Possible livelock in CapacityScheduler when RM is recovering apps</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2453">YARN-2453</a>.
-     Major bug reported by zhihai xu and fixed by zhihai xu <br>
-     <b>TestProportionalCapacityPreemptionPolicy fails with FairScheduler</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2452">YARN-2452</a>.
-     Major bug reported by zhihai xu and fixed by zhihai xu <br>
-     <b>TestRMApplicationHistoryWriter fails with FairScheduler</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2450">YARN-2450</a>.
-     Trivial bug reported by Ray Chiang and fixed by Ray Chiang <br>
-     <b>Fix typos in log messages</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2449">YARN-2449</a>.
-     Critical bug reported by Karam Singh and fixed by Varun Vasudev (timelineserver)<br>
-     <b>Timelineserver returns invalid Delegation token in secure kerberos enabled cluster when hadoop.http.filter.initializers are not set</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2448">YARN-2448</a>.
-     Major improvement reported by Varun Vasudev and fixed by Varun Vasudev <br>
-     <b>RM should expose the resource types considered during scheduling when AMs register</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2447">YARN-2447</a>.
-     Major bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
-     <b>RM web services app submission doesn't pass secrets correctly</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2446">YARN-2446</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Using TimelineNamespace to shield the entities of a user</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2440">YARN-2440</a>.
-     Major bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
-     <b>Cgroups should allow YARN containers to be limited to allocated cores</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2434">YARN-2434</a>.
-     Major sub-task reported by Jian He and fixed by Jian He <br>
-     <b>RM should not recover containers from previously failed attempt when AM restart is not enabled</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2431">YARN-2431</a>.
-     Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
-     <b>NM restart: cgroup is not removed for reacquired containers</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2424">YARN-2424</a>.
-     Blocker bug reported by Allen Wittenauer and fixed by Allen Wittenauer (nodemanager)<br>
-     <b>LCE should support non-cgroups, non-secure mode</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2411">YARN-2411</a>.
-     Major improvement reported by Ram Venkatesh and fixed by Ram Venkatesh (capacityscheduler)<br>
-     <b>[Capacity Scheduler] support simple user and group mappings to queues</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2409">YARN-2409</a>.
-     Critical bug reported by Nishan Shetty and fixed by Rohith (resourcemanager)<br>
-     <b>Active to StandBy transition does not stop rmDispatcher that causes 1 AsyncDispatcher thread leak. </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2406">YARN-2406</a>.
-     Major sub-task reported by Jian He and fixed by Tsuyoshi OZAWA <br>
-     <b>Move RM recovery related proto to yarn_server_resourcemanager_recovery.proto</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2405">YARN-2405</a>.
-     Major bug reported by Maysam Yabandeh and fixed by Tsuyoshi OZAWA <br>
-     <b>NPE in FairSchedulerAppsBlock</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2400">YARN-2400</a>.
-     Major bug reported by Jian He and fixed by Jian He <br>
-     <b>TestAMRestart fails intermittently</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2399">YARN-2399</a>.
-     Major improvement reported by Karthik Kambatla and fixed by Karthik Kambatla (fairscheduler)<br>
-     <b>FairScheduler: Merge AppSchedulable and FSSchedulerApp into FSAppAttempt</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2397">YARN-2397</a>.
-     Critical bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
-     <b>RM and TS web interfaces sometimes return request is a replay error in secure mode</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2395">YARN-2395</a>.
-     Major new feature reported by Ashwin Shankar and fixed by Wei Yan (fairscheduler)<br>
-     <b>FairScheduler: Preemption timeout should be configurable per queue</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2394">YARN-2394</a>.
-     Major new feature reported by Ashwin Shankar and fixed by Wei Yan (fairscheduler)<br>
-     <b>FairScheduler: Configure fairSharePreemptionThreshold per queue</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2393">YARN-2393</a>.
-     Major new feature reported by Ashwin Shankar and fixed by Wei Yan (fairscheduler)<br>
-     <b>FairScheduler: Add the notion of steady fair share</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2389">YARN-2389</a>.
-     Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler , fairscheduler)<br>
-     <b>Adding support for drainig a queue, ie killing all apps in the queue</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2388">YARN-2388</a>.
-     Major test reported by Zhijie Shen and fixed by Zhijie Shen <br>
-     <b>TestTimelineWebServices fails on trunk after HADOOP-10791</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2387">YARN-2387</a>.
-     Blocker bug reported by Mit Desai and fixed by Mit Desai <br>
-     <b>Resource Manager crashes with NPE due to lack of synchronization</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2378">YARN-2378</a>.
-     Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler)<br>
-     <b>Adding support for moving apps between queues in Capacity Scheduler</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2377">YARN-2377</a>.
-     Major improvement reported by Gera Shegalov and fixed by Gera Shegalov (nodemanager)<br>
-     <b>Localization exception stack traces are not passed as diagnostic info</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2374">YARN-2374</a>.
-     Major bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
-     <b>YARN trunk build failing TestDistributedShell.testDSShell</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2373">YARN-2373</a>.
-     Major bug reported by Larry McCay and fixed by Larry McCay <br>
-     <b>WebAppUtils Should Use configuration.getPassword for Accessing SSL Passwords</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2372">YARN-2372</a>.
-     Minor improvement reported by Fengdong Yu and fixed by Fengdong Yu (documentation)<br>
-     <b>There are Chinese Characters in the FairScheduler's document</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2370">YARN-2370</a>.
-     Trivial bug reported by Wenwu Peng and fixed by Wenwu Peng (resourcemanager)<br>
-     <b>Fix comment in o.a.h.y.server.resourcemanager.schedulerAppSchedulingInfo</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2363">YARN-2363</a>.
-     Major bug reported by Jason Lowe and fixed by Jason Lowe (resourcemanager)<br>
-     <b>Submitted applications occasionally lack a tracking URL</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2361">YARN-2361</a>.
-     Trivial improvement reported by zhihai xu and fixed by zhihai xu (resourcemanager)<br>
-     <b>RMAppAttempt state machine entries for KILLED state has duplicate event entries</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2360">YARN-2360</a>.
-     Major new feature reported by Ashwin Shankar and fixed by Ashwin Shankar (fairscheduler)<br>
-     <b>Fair Scheduler: Display dynamic fair share for queues on the scheduler page</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2359">YARN-2359</a>.
-     Critical bug reported by zhihai xu and fixed by zhihai xu (resourcemanager)<br>
-     <b>Application hangs when it fails to launch AM container </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2354">YARN-2354</a>.
-     Major sub-task reported by Jian He and fixed by Li Lu <br>
-     <b>DistributedShell may allocate more containers than client specified after it restarts</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2352">YARN-2352</a>.
-     Major improvement reported by Karthik Kambatla and fixed by Karthik Kambatla (scheduler)<br>
-     <b>FairScheduler: Collect metrics on duration of critical methods that affect performance</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2347">YARN-2347</a>.
-     Major sub-task reported by Junping Du and fixed by Junping Du <br>
-     <b>Consolidate RMStateVersion and NMDBSchemaVersion into StateVersion in yarn-server-common</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2343">YARN-2343</a>.
-     Trivial improvement reported by Li Lu and fixed by Li Lu <br>
-     <b>Improve error message on token expire exception</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2337">YARN-2337</a>.
-     Trivial improvement reported by zhihai xu and fixed by zhihai xu (resourcemanager)<br>
-     <b>ResourceManager sets ClientRMService in RMContext multiple times</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2323">YARN-2323</a>.
-     Minor improvement reported by Hong Zhiguo and fixed by Hong Zhiguo (fairscheduler)<br>
-     <b>FairShareComparator creates too many Resource objects</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2321">YARN-2321</a>.
-     Major bug reported by Leitao Guo and fixed by Leitao Guo (nodemanager)<br>
-     <b>NodeManager web UI can incorrectly report Pmem enforcement</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2317">YARN-2317</a>.
-     Major sub-task reported by Li Lu and fixed by Li Lu (documentation)<br>
-     <b>Update documentation about how to write YARN applications</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2314">YARN-2314</a>.
-     Critical bug reported by Jason Lowe and fixed by Jason Lowe (client)<br>
-     <b>ContainerManagementProtocolProxy can create thousands of threads for a large cluster</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2313">YARN-2313</a>.
-     Major bug reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (fairscheduler)<br>
-     <b>Livelock can occur in FairScheduler when there are lots of running apps</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2312">YARN-2312</a>.
-     Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
-     <b>Marking ContainerId#getId as deprecated</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2308">YARN-2308</a>.
-     Critical bug reported by Wangda Tan and fixed by chang li (resourcemanager , scheduler)<br>
-     <b>NPE happened when RM restart after CapacityScheduler queue configuration changed </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2302">YARN-2302</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Refactor TimelineWebServices</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2298">YARN-2298</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (client)<br>
-     <b>Move TimelineClient to yarn-common project</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2295">YARN-2295</a>.
-     Major sub-task reported by Li Lu and fixed by Li Lu <br>
-     <b>Refactor YARN distributed shell with existing public stable API</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2288">YARN-2288</a>.
-     Major sub-task reported by Junping Du and fixed by Junping Du (timelineserver)<br>
-     <b>Data persistent in timelinestore should be versioned</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2279">YARN-2279</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
-     <b>Add UTs to cover timeline server authentication</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2277">YARN-2277</a>.
-     Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
-     <b>Add Cross-Origin support to the ATS REST API</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2274">YARN-2274</a>.
-     Trivial improvement reported by Karthik Kambatla and fixed by Karthik Kambatla (fairscheduler)<br>
-     <b>FairScheduler: Add debug information about cluster capacity, availability and reservations</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2273">YARN-2273</a>.
-     Major bug reported by Andy Skelton and fixed by Wei Yan (fairscheduler , resourcemanager)<br>
-     <b>NPE in ContinuousScheduling thread when we lose a node</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2269">YARN-2269</a>.
-     Major bug reported by Yesha Vora and fixed by Craig Welch <br>
-     <b>External links need to be removed from YARN UI</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2264">YARN-2264</a>.
-     Major bug reported by Siddharth Seth and fixed by Li Lu <br>
-     <b>Race in DrainDispatcher can cause random test failures</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2260">YARN-2260</a>.
-     Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
-     <b>Add containers to launchedContainers list in RMNode on container recovery</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2252">YARN-2252</a>.
-     Major bug reported by Ratandeep Ratti and fixed by  (scheduler)<br>
-     <b>Intermittent failure of TestFairScheduler.testContinuousScheduling</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2251">YARN-2251</a>.
-     Major bug reported by Zhijie Shen and fixed by Zhijie Shen <br>
-     <b>Avoid negative elapsed time in JHS/MRAM web UI and services</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2249">YARN-2249</a>.
-     Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
-     <b>AM release request may be lost on RM restart</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2248">YARN-2248</a>.
-     Major sub-task reported by Janos Matyas and fixed by Janos Matyas (capacityscheduler)<br>
-     <b>Capacity Scheduler changes for moving apps between queues</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2244">YARN-2244</a>.
-     Critical bug reported by Anubhav Dhoot and fixed by Anubhav Dhoot (fairscheduler)<br>
-     <b>FairScheduler missing handling of containers for unknown application attempts </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2242">YARN-2242</a>.
-     Major sub-task reported by Li Lu and fixed by Li Lu <br>
-     <b>Improve exception information on AM launch crashes</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2237">YARN-2237</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
-     <b>MRAppMaster changes for AMRMToken roll-up</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2229">YARN-2229</a>.
-     Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
-     <b>ContainerId can overflow with RM restart</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2228">YARN-2228</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
-     <b>TimelineServer should load pseudo authentication filter when authentication = simple</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2219">YARN-2219</a>.
-     Major bug reported by Ashwin Shankar and fixed by Jian He (resourcemanager)<br>
-     <b>AMs and NMs can get exceptions after recovery but before scheduler knowns apps and app-attempts</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2214">YARN-2214</a>.
-     Major improvement reported by Ashwin Shankar and fixed by Ashwin Shankar (scheduler)<br>
-     <b>FairScheduler: preemptContainerPreCheck() in FSParentQueue delays convergence towards fairness</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2212">YARN-2212</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
-     <b>ApplicationMaster needs to find a way to update the AMRMToken periodically</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2211">YARN-2211</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
-     <b>RMStateStore needs to save AMRMToken master key for recovery when RM restart/failover happens </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2209">YARN-2209</a>.
-     Major improvement reported by Jian He and fixed by Jian He <br>
-     <b>Replace AM resync/shutdown command with corresponding exceptions</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2208">YARN-2208</a>.
-     Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
-     <b>AMRMTokenManager need to have a way to roll over AMRMToken</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2207">YARN-2207</a>.
-     Major task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
-     <b>Add ability to roll over AMRMToken</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2198">YARN-2198</a>.
-     Major improvement reported by Remus Rusanu and fixed by Remus Rusanu <br>
-     <b>Remove the need to run NodeManager as privileged account for Windows Secure Container Executor</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2197">YARN-2197</a>.
-     Minor improvement reported by Akira AJISAKA and fixed by Akira AJISAKA (documentation)<br>
-     <b>Add a link to YARN CHANGES.txt in the left side of doc</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2182">YARN-2182</a>.
-     Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
-     <b>Update ContainerId#toString() to avoid conflicts before and after RM restart</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2181">YARN-2181</a>.
-     Major bug reported by Wangda Tan and fixed by Wangda Tan (resourcemanager , webapp)<br>
-     <b>Add preemption info to RM Web UI and add logs when preemption occurs</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2174">YARN-2174</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
-     <b>Enabling HTTPs for the writer REST API of TimelineServer</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2161">YARN-2161</a>.
-     Major bug reported by Binglin Chang and fixed by Binglin Chang <br>
-     <b>Fix build on macosx: YARN parts</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2158">YARN-2158</a>.
-     Minor test reported by Ted Yu and fixed by Varun Vasudev <br>
-     <b>TestRMWebServicesAppsModification sometimes fails in trunk</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2153">YARN-2153</a>.
-     Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
-     <b>Ensure distributed shell work with RM work-preserving recovery</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2147">YARN-2147</a>.
-     Minor bug reported by Jason Lowe and fixed by Chen He (resourcemanager)<br>
-     <b>client lacks delegation token exception details when application submit fails</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2138">YARN-2138</a>.
-     Major bug reported by Jian He and fixed by Varun Saxena <br>
-     <b>Cleanup notifyDone* methods in RMStateStore</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2131">YARN-2131</a>.
-     Major new feature reported by Karthik Kambatla and fixed by Robert Kanter (resourcemanager)<br>
-     <b>Add a way to format the RMStateStore</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2102">YARN-2102</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
-     <b>More generalized timeline ACLs</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2088">YARN-2088</a>.
-     Major bug reported by Binglin Chang and fixed by Binglin Chang <br>
-     <b>Fix code bug in GetApplicationsRequestPBImpl#mergeLocalToBuilder</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2080">YARN-2080</a>.
-     Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (resourcemanager)<br>
-     <b>Admission Control: Integrate Reservation subsystem with ResourceManager</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2070">YARN-2070</a>.
-     Minor sub-task reported by Zhijie Shen and fixed by Robert Kanter <br>
-     <b>DistributedShell publishes unfriendly user information to the timeline server</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2045">YARN-2045</a>.
-     Major sub-task reported by Junping Du and fixed by Junping Du (nodemanager)<br>
-     <b>Data persisted in NM should be versioned</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2035">YARN-2035</a>.
-     Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles <br>
-     <b>FileSystemApplicationHistoryStore blocks RM and AHS while NN is in safemode</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2034">YARN-2034</a>.
-     Minor bug reported by Jason Lowe and fixed by Chen He (nodemanager)<br>
-     <b>Description for yarn.nodemanager.localizer.cache.target-size-mb is incorrect</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2033">YARN-2033</a>.
-     Major sub-task reported by Vinod Kumar Vavilapalli and fixed by Zhijie Shen <br>
-     <b>Merging generic-history into the Timeline Store</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2026">YARN-2026</a>.
-     Major bug reported by Ashwin Shankar and fixed by Ashwin Shankar (scheduler)<br>
-     <b>Fair scheduler: Consider only active queues for computing fairshare</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2013">YARN-2013</a>.
-     Major sub-task reported by Zhijie Shen and fixed by Tsuyoshi OZAWA (nodemanager)<br>
-     <b>The diagnostics is always the ExitCodeException stack when the container crashes</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2010">YARN-2010</a>.
-     Blocker bug reported by bc Wong and fixed by Karthik Kambatla (resourcemanager)<br>
-     <b>Handle app-recovery failures gracefully</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2008">YARN-2008</a>.
-     Major sub-task reported by Chen He and fixed by Craig Welch <br>
-     <b>CapacityScheduler may report incorrect queueMaxCap if there is hierarchy queue structure </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2001">YARN-2001</a>.
-     Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
-     <b>Threshold for RM to accept requests from AM after failover</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1994">YARN-1994</a>.
-     Major improvement reported by Arpit Agarwal and fixed by Craig Welch (nodemanager , resourcemanager , webapp)<br>
-     <b>Expose YARN/MR endpoints on multiple interfaces</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1972">YARN-1972</a>.
-     Major sub-task reported by Remus Rusanu and fixed by Remus Rusanu (nodemanager)<br>
-     <b>Implement secure Windows Container Executor</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1964">YARN-1964</a>.
-     Major new feature reported by Arun C Murthy and fixed by Abin Shahab <br>
-     <b>Create Docker analog of the LinuxContainerExecutor in YARN</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1959">YARN-1959</a>.
-     Major bug reported by Sandy Ryza and fixed by Anubhav Dhoot <br>
-     <b>Fix headroom calculation in FairScheduler</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1954">YARN-1954</a>.
-     Major improvement reported by Zhijie Shen and fixed by Tsuyoshi OZAWA (client)<br>
-     <b>Add waitFor to AMRMClient(Async)</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1922">YARN-1922</a>.
-     Major bug reported by Billie Rinaldi and fixed by Billie Rinaldi (nodemanager)<br>
-     <b>Process group remains alive after container process is killed externally</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1919">YARN-1919</a>.
-     Minor bug reported by Devaraj K and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
-     <b>Potential NPE in EmbeddedElectorService#stop</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1918">YARN-1918</a>.
-     Trivial improvement reported by Devaraj K and fixed by Anandha L Ranganathan <br>
-     <b>Typo in description and error message for 'yarn.resourcemanager.cluster-id'</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1915">YARN-1915</a>.
-     Blocker sub-task reported by Hitesh Shah and fixed by Jason Lowe <br>
-     <b>ClientToAMTokenMasterKey should be provided to AM at launch time</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1879">YARN-1879</a>.
-     Critical sub-task reported by Jian He and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
-     <b>Mark Idempotent/AtMostOnce annotations to ApplicationMasterProtocol for RM fail over</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1857">YARN-1857</a>.
-     Critical sub-task reported by Thomas Graves and fixed by Chen He (capacityscheduler)<br>
-     <b>CapacityScheduler headroom doesn't account for other AM's running</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1796">YARN-1796</a>.
-     Minor bug reported by Aaron T. Myers and fixed by Aaron T. Myers (nodemanager)<br>
-     <b>container-executor shouldn't require o-r permissions</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1779">YARN-1779</a>.
-     Blocker sub-task reported by Karthik Kambatla and fixed by Jian He (resourcemanager)<br>
-     <b>Handle AMRMTokens across RM failover</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1769">YARN-1769</a>.
-     Major improvement reported by Thomas Graves and fixed by Thomas Graves (capacityscheduler)<br>
-     <b>CapacityScheduler:  Improve reservations</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1712">YARN-1712</a>.
-     Major sub-task reported by Carlo Curino and fixed by Carlo Curino (capacityscheduler , resourcemanager)<br>
-     <b>Admission Control: plan follower</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1711">YARN-1711</a>.
-     Major sub-task reported by Carlo Curino and fixed by Carlo Curino <br>
-     <b>CapacityOverTimePolicy: a policy to enforce quotas over time for YARN-1709</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1710">YARN-1710</a>.
-     Major sub-task reported by Carlo Curino and fixed by Carlo Curino (resourcemanager)<br>
-     <b>Admission Control: agents to allocate reservation</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1709">YARN-1709</a>.
-     Major sub-task reported by Carlo Curino and fixed by Subru Krishnan (resourcemanager)<br>
-     <b>Admission Control: Reservation subsystem</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1708">YARN-1708</a>.
-     Major sub-task reported by Carlo Curino and fixed by Subru Krishnan (resourcemanager)<br>
-     <b>Add a public API to reserve resources (part of YARN-1051)</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1707">YARN-1707</a>.
-     Major sub-task reported by Carlo Curino and fixed by Carlo Curino (capacityscheduler)<br>
-     <b>Making the CapacityScheduler more dynamic</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1372">YARN-1372</a>.
-     Major sub-task reported by Bikas Saha and fixed by Anubhav Dhoot (resourcemanager)<br>
-     <b>Ensure all completed containers are reported to the AMs across RM restart</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1370">YARN-1370</a>.
-     Major sub-task reported by Bikas Saha and fixed by Anubhav Dhoot (resourcemanager)<br>
-     <b>Fair scheduler to re-populate container allocation state</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1367">YARN-1367</a>.
-     Major sub-task reported by Bikas Saha and fixed by Anubhav Dhoot (resourcemanager)<br>
-     <b>After restart NM should resync with the RM without killing containers</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1354">YARN-1354</a>.
-     Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
-     <b>Recover applications upon nodemanager restart</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1342">YARN-1342</a>.
-     Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
-     <b>Recover container tokens upon nodemanager restart</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1341">YARN-1341</a>.
-     Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
-     <b>Recover NMTokens upon nodemanager restart</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1337">YARN-1337</a>.
-     Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
-     <b>Recover containers upon nodemanager restart</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1326">YARN-1326</a>.
-     Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
-     <b>RM should log using RMStore at startup time</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1250">YARN-1250</a>.
-     Major sub-task reported by Vinod Kumar Vavilapalli and fixed by Zhijie Shen <br>
-     <b>Generic history service should support application-acls</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1063">YARN-1063</a>.
-     Major sub-task reported by Kyle Leckie and fixed by Remus Rusanu (nodemanager)<br>
-     <b>Winutils needs ability to create task as domain user</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1051">YARN-1051</a>.
-     Major improvement reported by Carlo Curino and fixed by Carlo Curino (capacityscheduler , resourcemanager , scheduler)<br>
-     <b>YARN Admission Control/Planner: enhancing the resource allocation model with time.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-668">YARN-668</a>.
-     Blocker sub-task reported by Siddharth Seth and fixed by Junping Du <br>
-     <b>TokenIdentifier serialization should consider Unknown fields</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-611">YARN-611</a>.
-     Major sub-task reported by Chris Riccomini and fixed by Xuan Gong (resourcemanager)<br>
-     <b>Add an AM retry count reset window to YARN RM</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-415">YARN-415</a>.
-     Major new feature reported by Kendall Thrapp and fixed by Eric Payne (resourcemanager)<br>
-     <b>Capture aggregate memory allocation at the app-level for chargeback</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-90">YARN-90</a>.
-     Major sub-task reported by Ravi Gummadi and fixed by Varun Vasudev (nodemanager)<br>
-     <b>NodeManager should identify failed disks becoming good again</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6156">MAPREDUCE-6156</a>.
-     Blocker bug reported by Sidharta Seethana and fixed by Junping Du <br>
-     <b>Fetcher - connect() doesn't handle connection refused correctly </b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6142">MAPREDUCE-6142</a>.
-     Critical sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
-     <b>Test failure in TestJobHistoryEventHandler and TestMRTimelineEventHandling</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6126">MAPREDUCE-6126</a>.
-     Major bug reported by Junping Du and fixed by Junping Du <br>
-     <b>(Rumen) Rumen tool returns error "ava.lang.IllegalArgumentException: JobBuilder.process(HistoryEvent): unknown event type"</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6125">MAPREDUCE-6125</a>.
-     Major bug reported by Mit Desai and fixed by Mit Desai (test)<br>
-     <b>TestContainerLauncherImpl sometimes fails</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6123">MAPREDUCE-6123</a>.
-     Trivial bug reported by Chris Nauroth and fixed by Chris Nauroth (test)<br>
-     <b>TestCombineFileInputFormat incorrectly starts 2 MiniDFSCluster instances.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6122">MAPREDUCE-6122</a>.
-     Trivial bug reported by Chris Nauroth and fixed by Chris Nauroth (test)<br>
-     <b>TestLineRecordReader may fail due to test data files checked out of git with incorrect line endings.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6115">MAPREDUCE-6115</a>.
-     Minor test reported by Ted Yu and fixed by Binglin Chang <br>
-     <b>TestPipeApplication#testSubmitter fails in trunk</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6109">MAPREDUCE-6109</a>.
-     Trivial bug reported by Charles Lamb and fixed by Charles Lamb (distcp)<br>
-     <b>Fix minor typo in distcp -p usage text</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6104">MAPREDUCE-6104</a>.
-     Major bug reported by Mit Desai and fixed by Mit Desai <br>
-     <b>TestJobHistoryParsing.testPartialJob fails in branch-2</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6095">MAPREDUCE-6095</a>.
-     Major bug reported by Gera Shegalov and fixed by Gera Shegalov (applicationmaster , distributed-cache)<br>
-     <b>Enable DistributedCache for uber-mode Jobs</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6094">MAPREDUCE-6094</a>.
-     Minor bug reported by Sangjin Lee and fixed by Akira AJISAKA (test)<br>
-     <b>TestMRCJCFileInputFormat.testAddInputPath() fails on trunk</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6093">MAPREDUCE-6093</a>.
-     Trivial bug reported by Charles Lamb and fixed by Charles Lamb (distcp , documentation)<br>
-     <b>minor distcp doc edits</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6091">MAPREDUCE-6091</a>.
-     Major bug reported by Sangjin Lee and fixed by Sangjin Lee (client)<br>
-     <b>YARNRunner.getJobStatus() fails with ApplicationNotFoundException if the job rolled off the RM view</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6090">MAPREDUCE-6090</a>.
-     Major bug reported by Robert Kanter and fixed by Robert Kanter (client)<br>
-     <b>mapred hsadmin getGroups fails to connect in some cases</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6086">MAPREDUCE-6086</a>.
-     Major improvement reported by zhihai xu and fixed by zhihai xu (security)<br>
-     <b>mapreduce.job.credentials.binary should allow all URIs</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6075">MAPREDUCE-6075</a>.
-     Major bug reported by Jason Lowe and fixed by Jason Lowe (jobhistoryserver)<br>
-     <b>HistoryServerFileSystemStateStore can create zero-length files</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6073">MAPREDUCE-6073</a>.
-     Trivial bug reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (documentation)<br>
-     <b>Description of mapreduce.job.speculative.slowtaskthreshold in mapred-default should be moved into description tags</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6072">MAPREDUCE-6072</a>.
-     Minor improvement reported by Akira AJISAKA and fixed by Akira AJISAKA (documentation)<br>
-     <b>Remove INSTALL document</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6071">MAPREDUCE-6071</a>.
-     Trivial improvement reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (client)<br>
-     <b>JobImpl#makeUberDecision doesn't log that Uber mode is disabled because of too much CPUs</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6070">MAPREDUCE-6070</a>.
-     Trivial improvement reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (documentation)<br>
-     <b>yarn.app.am.resource.mb/cpu-vcores affects uber mode but is not documented</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6063">MAPREDUCE-6063</a>.
-     Major bug reported by zhihai xu and fixed by zhihai xu (mrv1 , mrv2)<br>
-     <b>In sortAndSpill of MapTask.java, size is calculated wrongly when bufend &lt; bufstart.</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6052">MAPREDUCE-6052</a>.
-     Major bug reported by Junping Du and fixed by Junping Du <br>
-     <b>Support overriding log4j.properties per job</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6051">MAPREDUCE-6051</a>.
-     Trivial bug reported by Ray Chiang and fixed by Ray Chiang <br>
-     <b>Fix typos in log messages</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6048">MAPREDUCE-6048</a>.
-     Minor test reported by Ted Yu and fixed by Varun Vasudev <br>
-     <b>TestJavaSerialization fails in trunk build</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6044">MAPREDUCE-6044</a>.
-     Major bug reported by Zhijie Shen and fixed by Zhijie Shen (jobhistoryserver)<br>
-     <b>Fully qualified intermediate done directory will break per-user dir creation on Windows</b><br>
-     <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6036">MAPREDUCE-6036</a>.
-     Major bug reported by Mit Desai and fixed by chang li 

<TRUNCATED>

[49/50] [abbrv] hadoop git commit: YARN-1462. Correct fix version from branch-2.7.1 to branch-2.8 in CHANGES.txt

Posted by zj...@apache.org.
YARN-1462. Correct fix version from branch-2.7.1 to branch-2.8 in
CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7505e35e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7505e35e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7505e35e

Branch: refs/heads/YARN-2928
Commit: 7505e35e56a0f6e91d50f00d9c915aa46dca4fff
Parents: 850e488
Author: Xuan <xg...@apache.org>
Authored: Mon Jun 1 11:39:00 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:22 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7505e35e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 807165f..543d9cf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -374,6 +374,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3467. Expose allocatedMB, allocatedVCores, and runningContainers metrics on 
     running Applications in RM Web UI. (Anubhav Dhoot via kasha)
 
+    YARN-1462. Made RM write application tags to timeline server and exposed them
+    to users via generic history web UI and REST API. (Xuan Gong via zjshen)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not
@@ -584,9 +587,6 @@ Release 2.7.1 - UNRELEASED
     YARN-3723. Need to clearly document primaryFilter and otherInfo value type.
     (Zhijie Shen via xgong)
 
-    YARN-1462. Made RM write application tags to timeline server and exposed them
-    to users via generic history web UI and REST API. (Xuan Gong via zjshen)
-
   OPTIMIZATIONS
 
   BUG FIXES


[18/50] [abbrv] hadoop git commit: HDFS-8429. Avoid stuck threads if there is an error in DomainSocketWatcher that stops the thread. (zhouyingchao via cmccabe)

Posted by zj...@apache.org.
HDFS-8429. Avoid stuck threads if there is an error in DomainSocketWatcher that stops the thread.  (zhouyingchao via cmccabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d5c0d4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d5c0d4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d5c0d4a

Branch: refs/heads/YARN-2928
Commit: 3d5c0d4a2a8d733bcd6a95a85932a2a6898c20b4
Parents: 61510b9
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu May 28 11:52:28 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +
 .../hadoop/net/unix/DomainSocketWatcher.java    | 21 +++++-
 .../hadoop/net/unix/DomainSocketWatcher.c       |  2 +-
 .../net/unix/TestDomainSocketWatcher.java       | 75 ++++++++++++++++++++
 4 files changed, 99 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5c0d4a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 69f5aa7..51eff78 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -782,6 +782,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12035. shellcheck plugin displays a wrong version potentially
     (Kengo Seki via aw)
 
+    HDFS-8429. Avoid stuck threads if there is an error in DomainSocketWatcher
+    that stops the thread.  (zhouyingchao via cmccabe)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5c0d4a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
index 5648ae1..ad2fbfb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
@@ -470,6 +470,7 @@ public final class DomainSocketWatcher implements Closeable {
               // Handle pending additions (before pending removes).
               for (Iterator<Entry> iter = toAdd.iterator(); iter.hasNext(); ) {
                 Entry entry = iter.next();
+                iter.remove();
                 DomainSocket sock = entry.getDomainSocket();
                 Entry prevEntry = entries.put(sock.fd, entry);
                 Preconditions.checkState(prevEntry == null,
@@ -479,7 +480,6 @@ public final class DomainSocketWatcher implements Closeable {
                   LOG.trace(this + ": adding fd " + sock.fd);
                 }
                 fdSet.add(sock.fd);
-                iter.remove();
               }
               // Handle pending removals
               while (true) {
@@ -525,6 +525,25 @@ public final class DomainSocketWatcher implements Closeable {
           }
           entries.clear();
           fdSet.close();
+          closed = true;
+          if (!(toAdd.isEmpty() && toRemove.isEmpty())) {
+            // Items in toAdd might not be added to entries, handle it here
+            for (Iterator<Entry> iter = toAdd.iterator(); iter.hasNext();) {
+              Entry entry = iter.next();
+              entry.getDomainSocket().refCount.unreference();
+              entry.getHandler().handle(entry.getDomainSocket());
+              IOUtils.cleanup(LOG, entry.getDomainSocket());
+              iter.remove();
+            }
+            // Items in toRemove might not be really removed, handle it here
+            while (true) {
+              Map.Entry<Integer, DomainSocket> entry = toRemove.firstEntry();
+              if (entry == null)
+                break;
+              sendCallback("close", entries, fdSet, entry.getValue().fd);
+            }
+          }
+          processedCond.signalAll();
         } finally {
           lock.unlock();
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5c0d4a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c
index 596601b..82e6af5 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c
@@ -212,7 +212,7 @@ done:
   free(carr);
   if (jthr) {
     (*env)->DeleteLocalRef(env, jarr);
-    jarr = NULL;
+    (*env)->Throw(env, jthr);
   }
   return jarr;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5c0d4a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
index 4b0e2a8..4cc86a7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse;
 import java.util.ArrayList;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -181,6 +182,80 @@ public class TestDomainSocketWatcher {
     watcher.close();
   }
 
+  @Test(timeout = 300000)
+  public void testStressInterruption() throws Exception {
+    final int SOCKET_NUM = 250;
+    final ReentrantLock lock = new ReentrantLock();
+    final DomainSocketWatcher watcher = newDomainSocketWatcher(10);
+    final ArrayList<DomainSocket[]> pairs = new ArrayList<DomainSocket[]>();
+    final AtomicInteger handled = new AtomicInteger(0);
+
+    final Thread adderThread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        try {
+          for (int i = 0; i < SOCKET_NUM; i++) {
+            DomainSocket pair[] = DomainSocket.socketpair();
+            watcher.add(pair[1], new DomainSocketWatcher.Handler() {
+              @Override
+              public boolean handle(DomainSocket sock) {
+                handled.incrementAndGet();
+                return true;
+              }
+            });
+            lock.lock();
+            try {
+              pairs.add(pair);
+            } finally {
+              lock.unlock();
+            }
+            TimeUnit.MILLISECONDS.sleep(1);
+          }
+        } catch (Throwable e) {
+          LOG.error(e);
+          throw new RuntimeException(e);
+        }
+      }
+    });
+
+    final Thread removerThread = new Thread(new Runnable() {
+      @Override
+      public void run() {
+        final Random random = new Random();
+        try {
+          while (handled.get() != SOCKET_NUM) {
+            lock.lock();
+            try {
+              if (!pairs.isEmpty()) {
+                int idx = random.nextInt(pairs.size());
+                DomainSocket pair[] = pairs.remove(idx);
+                if (random.nextBoolean()) {
+                  pair[0].close();
+                } else {
+                  watcher.remove(pair[1]);
+                }
+                TimeUnit.MILLISECONDS.sleep(1);
+              }
+            } finally {
+              lock.unlock();
+            }
+          }
+        } catch (Throwable e) {
+          LOG.error(e);
+          throw new RuntimeException(e);
+        }
+      }
+    });
+
+    adderThread.start();
+    removerThread.start();
+    TimeUnit.MILLISECONDS.sleep(100);
+    watcher.watcherThread.interrupt();
+    Uninterruptibles.joinUninterruptibly(adderThread);
+    Uninterruptibles.joinUninterruptibly(removerThread);
+    Uninterruptibles.joinUninterruptibly(watcher.watcherThread);
+  }
+
   /**
    * Creates a new DomainSocketWatcher and tracks its thread for termination due
    * to an unexpected exception.  At the end of each test, if there was an


[44/50] [abbrv] hadoop git commit: Revert "YARN-2900. Application (Attempt and Container) Not Found in AHS results"

Posted by zj...@apache.org.
Revert "YARN-2900. Application (Attempt and Container) Not Found in AHS results"

This reverts commit 06f8e9cabaf3c05cd7d16215cff47265ea773f39.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00ccf736
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00ccf736
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00ccf736

Branch: refs/heads/YARN-2928
Commit: 00ccf736966b2980fd34756ad30f5f84d48efd9d
Parents: c0ecae3
Author: Xuan <xg...@apache.org>
Authored: Sun May 31 00:26:04 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:13:00 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 -
 .../timeline/webapp/TimelineWebServices.java    |  2 +
 .../TestApplicationHistoryClientService.java    | 70 +-------------------
 .../hadoop/yarn/server/webapp/WebServices.java  | 27 +++-----
 4 files changed, 13 insertions(+), 89 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/00ccf736/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 45c4f68..6f5faa9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -694,9 +694,6 @@ Release 2.7.1 - UNRELEASED
     YARN-3686. CapacityScheduler should trim default_node_label_expression. 
     (Sunil G via wangda)
 
-    YARN-2900. Application (Attempt and Container) Not Found in AHS results
-    in InternalServer Error (500). (Zhijie Shen and Mit Desai via xgong)
-
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00ccf736/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
index 40e9fed..90f4d39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
@@ -47,6 +47,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.VersionInfo;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@@ -60,6 +61,7 @@ import org.apache.hadoop.yarn.server.timeline.NameValuePair;
 import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
 import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 import org.apache.hadoop.yarn.webapp.ForbiddenException;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00ccf736/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index d04903c..764ecb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -41,9 +41,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
@@ -57,13 +54,12 @@ public class TestApplicationHistoryClientService {
 
   private static ApplicationHistoryClientService clientService;
   private static TimelineDataManager dataManager;
-  private final static int MAX_APPS = 2;
 
   @BeforeClass
   public static void setup() throws Exception {
     Configuration conf = new YarnConfiguration();
     TimelineStore store =
-        TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
+        TestApplicationHistoryManagerOnTimelineStore.createStore(2);
     TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
     dataManager =
         new TimelineDataManager(store, aclsManager);
@@ -76,70 +72,6 @@ public class TestApplicationHistoryClientService {
   }
 
   @Test
-  public void testApplicationNotFound() throws IOException, YarnException {
-    ApplicationId appId = null;
-    appId = ApplicationId.newInstance(0, MAX_APPS + 1);
-    GetApplicationReportRequest request =
-        GetApplicationReportRequest.newInstance(appId);
-    try {
-      @SuppressWarnings("unused")
-      GetApplicationReportResponse response =
-          clientService.getApplicationReport(request);
-      Assert.fail("Exception should have been thrown before we reach here.");
-    } catch (ApplicationNotFoundException e) {
-      //This exception is expected.
-      Assert.assertTrue(e.getMessage().contains(
-          "doesn't exist in the timeline store"));
-    } catch (Exception e) {
-      Assert.fail("Undesired exception caught");
-    }
-  }
-
-  @Test
-  public void testApplicationAttemptNotFound() throws IOException, YarnException {
-    ApplicationId appId = ApplicationId.newInstance(0, 1);
-    ApplicationAttemptId appAttemptId =
-        ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
-    GetApplicationAttemptReportRequest request =
-        GetApplicationAttemptReportRequest.newInstance(appAttemptId);
-    try {
-      @SuppressWarnings("unused")
-      GetApplicationAttemptReportResponse response =
-          clientService.getApplicationAttemptReport(request);
-      Assert.fail("Exception should have been thrown before we reach here.");
-    } catch (ApplicationAttemptNotFoundException e) {
-      //This Exception is expected
-      System.out.println(e.getMessage());
-      Assert.assertTrue(e.getMessage().contains(
-          "doesn't exist in the timeline store"));
-    } catch (Exception e) {
-      Assert.fail("Undesired exception caught");
-    }
-  }
-
-  @Test
-  public void testContainerNotFound() throws IOException, YarnException {
-   ApplicationId appId = ApplicationId.newInstance(0, 1);
-   ApplicationAttemptId appAttemptId =
-       ApplicationAttemptId.newInstance(appId, 1);
-   ContainerId containerId = ContainerId.newContainerId(appAttemptId,
-       MAX_APPS + 1);
-   GetContainerReportRequest request =
-       GetContainerReportRequest.newInstance(containerId);
-   try {
-   @SuppressWarnings("unused")
-   GetContainerReportResponse response =
-       clientService.getContainerReport(request);
-   } catch (ContainerNotFoundException e) {
-     //This exception is expected
-     Assert.assertTrue(e.getMessage().contains(
-         "doesn't exist in the timeline store"));
-   }  catch (Exception e) {
-      Assert.fail("Undesired exception caught");
-   }
- }
-
-  @Test
   public void testApplicationReport() throws IOException, YarnException {
     ApplicationId appId = null;
     appId = ApplicationId.newInstance(0, 1);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00ccf736/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index ae27d45..8404719 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -47,9 +47,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
-import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@@ -480,21 +477,17 @@ public class WebServices {
 
   private static void rewrapAndThrowException(Exception e) {
     if (e instanceof UndeclaredThrowableException) {
-      rewrapAndThrowThrowable(e.getCause());
-    } else {
-      rewrapAndThrowThrowable(e);
-    }
-  }
-
-  private static void rewrapAndThrowThrowable(Throwable t) {
-    if (t instanceof AuthorizationException) {
-      throw new ForbiddenException(t);
-    } if (t instanceof ApplicationNotFoundException ||
-        t instanceof ApplicationAttemptNotFoundException ||
-        t instanceof ContainerNotFoundException) {
-      throw new NotFoundException(t);
+      if (e.getCause() instanceof AuthorizationException) {
+        throw new ForbiddenException(e.getCause());
+      } else {
+        throw new WebApplicationException(e.getCause());
+      }
     } else {
-      throw new WebApplicationException(t);
+      if (e instanceof AuthorizationException) {
+        throw new ForbiddenException(e);
+      } else {
+        throw new WebApplicationException(e);
+      }
     }
   }
 


[50/50] [abbrv] hadoop git commit: HDFS-8256. '-storagepolicies , -blockId , -replicaDetails ' options are missed out in usage and from documentation (Contributed by J.Andreina)

Posted by zj...@apache.org.
HDFS-8256. '-storagepolicies , -blockId ,-replicaDetails ' options are missed out in usage and from documentation (Contributed by J.Andreina)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e124808
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e124808
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e124808

Branch: refs/heads/YARN-2928
Commit: 2e1248088731e53bc085471c4799a23097e82a05
Parents: 77c1692
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 2 16:07:18 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:23 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt                 | 3 +++
 .../src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java   | 9 +++++----
 .../hadoop-hdfs/src/site/markdown/HDFSCommands.md           | 6 +++++-
 3 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e124808/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1f7a36a..fa28a16 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -831,6 +831,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8490. Typo in trace enabled log in ExceptionHandler of WebHDFS.
     (Archana T via ozawa)
 
+    HDFS-8256. "-storagepolicies , -blockId ,-replicaDetails " options are missed
+    out in usage and from documentation (J.Andreina via vinayakumarb)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e124808/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 67386ac..33033e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -78,8 +78,9 @@ public class DFSck extends Configured implements Tool {
   private static final String USAGE = "Usage: hdfs fsck <path> "
       + "[-list-corruptfileblocks | "
       + "[-move | -delete | -openforwrite] "
-      + "[-files [-blocks [-locations | -racks]]]] "
-      + "[-includeSnapshots] [-showprogress]\n"
+      + "[-files [-blocks [-locations | -racks | -replicaDetails]]]] "
+      + "[-includeSnapshots] [-showprogress] "
+      + "[-storagepolicies] [-blockId <blk_Id>]\n"
       + "\t<path>\tstart checking from this path\n"
       + "\t-move\tmove corrupted files to /lost+found\n"
       + "\t-delete\tdelete corrupted files\n"
@@ -94,12 +95,12 @@ public class DFSck extends Configured implements Tool {
       + "\t-files -blocks -locations\tprint out locations for every block\n"
       + "\t-files -blocks -racks" 
       + "\tprint out network topology for data-node locations\n"
+      + "\t-files -blocks -replicaDetails\tprint out each replica details \n"
       + "\t-storagepolicies\tprint out storage policy summary for the blocks\n"
       + "\t-showprogress\tshow progress in output. Default is OFF (no progress)\n"
       + "\t-blockId\tprint out which file this blockId belongs to, locations"
       + " (nodes, racks) of this block, and other diagnostics info"
-      + " (under replicated, corrupted or not, etc)\n"
-      + "\t-replicaDetails\tprint out each replica details \n\n"
+      + " (under replicated, corrupted or not, etc)\n\n"
       + "Please Note:\n"
       + "\t1. By default fsck ignores files opened for write, "
       + "use -openforwrite to report such files. They are usually "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e124808/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 2ceff89..2121958 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -99,8 +99,9 @@ Usage:
        hdfs fsck <path>
               [-list-corruptfileblocks |
               [-move | -delete | -openforwrite]
-              [-files [-blocks [-locations | -racks]]]
+              [-files [-blocks [-locations | -racks | -replicaDetails]]]
               [-includeSnapshots] [-showprogress]
+              [-storagepolicies] [-blockId <blk_Id>]
 
 | COMMAND\_OPTION | Description |
 |:---- |:---- |
@@ -110,11 +111,14 @@ Usage:
 | `-files` `-blocks` | Print out the block report |
 | `-files` `-blocks` `-locations` | Print out locations for every block. |
 | `-files` `-blocks` `-racks` | Print out network topology for data-node locations. |
+| `-files` `-blocks` `-replicaDetails` | Print out each replica details. |
 | `-includeSnapshots` | Include snapshot data if the given path indicates a snapshottable directory or there are snapshottable directories under it. |
 | `-list-corruptfileblocks` | Print out list of missing blocks and files they belong to. |
 | `-move` | Move corrupted files to /lost+found. |
 | `-openforwrite` | Print out files opened for write. |
 | `-showprogress` | Print out dots for progress in output. Default is OFF (no progress). |
+| `-storagepolicies` | Print out storage policy summary for the blocks. |
+| `-blockId` | Print out information about the block. |
 
 Runs the HDFS filesystem checking utility. See [fsck](./HdfsUserGuide.html#fsck) for more info.
 


[12/50] [abbrv] hadoop git commit: YARN-3723. Need to clearly document primaryFilter and otherInfo value type. Contributed by Zhijie Shen

Posted by zj...@apache.org.
YARN-3723. Need to clearly document primaryFilter and otherInfo value
type. Contributed by Zhijie Shen


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de39fedb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de39fedb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de39fedb

Branch: refs/heads/YARN-2928
Commit: de39fedb5ca44b62aaef7400747e2a1abc312f41
Parents: ac3000a
Author: Xuan <xg...@apache.org>
Authored: Thu May 28 10:19:36 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +++
 .../src/site/markdown/TimelineServer.md         | 20 ++++++++++++++++----
 2 files changed, 19 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/de39fedb/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5326f2d..1dd4ee3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -566,6 +566,9 @@ Release 2.7.1 - UNRELEASED
     YARN-3539. Updated timeline server documentation and marked REST APIs evolving.
     (Steve Loughran via zjshen)
 
+    YARN-3723. Need to clearly document primaryFilter and otherInfo value type.
+    (Zhijie Shen via xgong)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/de39fedb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 5a18fb2..90f9512 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -648,11 +648,20 @@ Use the following URI to obtain all the entity objects of a given
   the set of fields contains `LAST_EVENT_ONLY` and not `EVENTS`, the most recent
   event for each entity is retrieved. If null, retrieves all fields.
 
+Note that the value of the key/value pair for `primaryFilter` and
+`secondaryFilters` parameters can be of different data types, and matching is
+data type sensitive. Users need to format the value properly. For example, `123`
+and `"123"` means an integer and a string respectively. If the entity has a
+string `"123"` for `primaryFilter`, but the parameter is set to the integer
+`123`, the entity will not be matched. Similarly, `true` means a boolean while
+`"true"` means a string. In general, the value will be casted as a certain Java
+type in consistent with `jackson` library parsing a JSON clip.
+
 ### Elements of the `entities` (Timeline Entity List) Object
 
 When you make a request for the list of timeline entities, the information
 will be returned as a collection of container objects. See also
-{{Timeline Entity}} for syntax of the timeline entity object.
+`Timeline Entity` for syntax of the timeline entity object.
 
 | Item | Data Type | Description|
 |:---- |:---- |:---- |
@@ -751,7 +760,9 @@ Use the following URI to obtain the entity object identified by the
 
 ### Elements of the `entity` (Timeline Entity) Object:
 
-See also {{Timeline Event List}} for syntax of the timeline event object.
+See also `Timeline Event List` for syntax of the timeline event object. Note
+that `value` of `primaryfilters` and `otherinfo` is an Object instead of a
+String.
 
 
 | Item | Data Type | Description|
@@ -841,7 +852,8 @@ will be returned as a collection of event objects.
 |:---- |:---- |:---- |
 | `events` | array of timeline event objects(JSON) | The collection of timeline event objects |
 
-Below is the elements of a single event object.
+Below is the elements of a single event object.  Note that `value` of
+`eventinfo` and `otherinfo` is an Object instead of a String.
 
 
 | Item | Data Type | Description|
@@ -1547,7 +1559,7 @@ None
 ### Elements of the `containers` (Container List) Object
 
 When you make a request for the list of containers, the information will be
-returned as a collection of container objects. See also {{Container}} for
+returned as a collection of container objects. See also `Container` for
 syntax of the container object.
 
 | Item | Data Type   | Description |


[09/50] [abbrv] hadoop git commit: YARN-3700. Made generic history service load a number of latest applications according to the parameter or the configuration. Contributed by Xuan Gong.

Posted by zj...@apache.org.
YARN-3700. Made generic history service load a number of latest applications according to the parameter or the configuration. Contributed by Xuan Gong.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81dd9e80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81dd9e80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81dd9e80

Branch: refs/heads/YARN-2928
Commit: 81dd9e80b32b89cbee312b7716b20da7dc8c06b5
Parents: 6fdef78
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed May 27 16:51:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:53 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +++
 .../hadoop/yarn/conf/YarnConfiguration.java     |  9 +++++++
 .../hadoop/yarn/webapp/YarnWebParams.java       |  1 +
 .../src/main/resources/yarn-default.xml         |  9 +++++++
 .../ApplicationHistoryClientService.java        |  2 +-
 .../ApplicationHistoryManager.java              | 12 ++++++---
 .../ApplicationHistoryManagerImpl.java          |  2 +-
 ...pplicationHistoryManagerOnTimelineStore.java | 13 +++++++---
 .../TestApplicationHistoryClientService.java    | 26 +++++++++++++++++---
 ...pplicationHistoryManagerOnTimelineStore.java | 12 +++++----
 .../hadoop/yarn/server/webapp/AppsBlock.java    |  7 +++++-
 .../hadoop/yarn/server/webapp/WebServices.java  | 17 ++++---------
 .../src/site/markdown/TimelineServer.md         |  1 +
 13 files changed, 83 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 54b33bc..5ce8c63 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -362,6 +362,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3581. Deprecate -directlyAccessNodeLabelStore in RMAdminCLI. 
     (Naganarasimha G R via wangda)
 
+    YARN-3700. Made generic history service load a number of latest applications
+    according to the parameter or the configuration. (Xuan Gong via zjshen)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index c84cc76..6892e2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1483,6 +1483,15 @@ public class YarnConfiguration extends Configuration {
   public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
       "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
 
+  /**
+   * Defines the max number of applications could be fetched using
+   * REST API or application history protocol and shown in timeline
+   * server web ui.
+   */
+  public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+      APPLICATION_HISTORY_PREFIX + "max-applications";
+  public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+
   /** Timeline service store class */
   public static final String TIMELINE_SERVICE_STORE =
       TIMELINE_SERVICE_PREFIX + "store-class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 1200690..199f4cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -33,6 +33,7 @@ public interface YarnWebParams {
   String ENTITY_STRING = "entity.string";
   String APP_OWNER = "app.owner";
   String APP_STATE = "app.state";
+  String APPS_NUM = "apps.num";
   String QUEUE_NAME = "queue.name";
   String NODE_STATE = "node.state";
   String NODE_LABEL = "node.label";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 74a5d1d..c7145b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1404,6 +1404,15 @@
   </property>
 
   <property>
+    <description>
+      Defines the max number of applications could be fetched using REST API or
+      application history protocol and shown in timeline server web ui.
+    </description>
+    <name>yarn.timeline-service.generic-application-history.max-applications</name>
+    <value>10000</value>
+  </property>
+
+  <property>
     <description>Store class name for timeline store.</description>
     <name>yarn.timeline-service.store-class</name>
     <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index e64ca14..7427926 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -186,7 +186,7 @@ public class ApplicationHistoryClientService extends AbstractService implements
           IOException {
     GetApplicationsResponse response =
         GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
-          history.getAllApplications().values()));
+          history.getApplications(request.getLimit()).values()));
     return response;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
index 041c31b..c218eec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
@@ -51,16 +51,20 @@ public interface ApplicationHistoryManager {
       IOException;
 
   /**
-   * This method returns all Application {@link ApplicationReport}s
-   * 
+   * This method returns the given number of Application
+   * {@link ApplicationReport}s.
+   *
+   * @param appsNum
+   *
    * @return map of {@link ApplicationId} to {@link ApplicationReport}s.
    * @throws YarnException
    * @throws IOException
    */
   @Public
   @Unstable
-  Map<ApplicationId, ApplicationReport> getAllApplications()
-      throws YarnException, IOException;
+  Map<ApplicationId, ApplicationReport>
+      getApplications(long appsNum) throws YarnException,
+          IOException;
 
   /**
    * Application can have multiple application attempts

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index c7cf07b..479858f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -98,7 +98,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
   }
 
   @Override
-  public Map<ApplicationId, ApplicationReport> getAllApplications()
+  public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
       throws IOException {
     Map<ApplicationId, ApplicationHistoryData> histData =
         historyStore.getAllApplications();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 2429a68..d3e7b5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -78,6 +79,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
   private TimelineDataManager timelineDataManager;
   private ApplicationACLsManager aclsManager;
   private String serverHttpAddress;
+  private long maxLoadedApplications;
 
   public ApplicationHistoryManagerOnTimelineStore(
       TimelineDataManager timelineDataManager,
@@ -91,6 +93,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
   protected void serviceInit(Configuration conf) throws Exception {
     serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
         WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
+    maxLoadedApplications =
+        conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
+          YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
     super.serviceInit(conf);
   }
 
@@ -101,12 +106,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
   }
 
   @Override
-  public Map<ApplicationId, ApplicationReport> getAllApplications()
+  public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
       throws YarnException, IOException {
     TimelineEntities entities = timelineDataManager.getEntities(
-        ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
-        null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
-        UserGroupInformation.getLoginUser());
+        ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
+        null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
+        EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
     Map<ApplicationId, ApplicationReport> apps =
         new LinkedHashMap<ApplicationId, ApplicationReport>();
     if (entities != null && entities.getEntities() != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index d03b26d..ad03718 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -53,6 +53,7 @@ import org.junit.Test;
 public class TestApplicationHistoryClientService {
 
   private static ApplicationHistoryClientService clientService;
+  private static TimelineDataManager dataManager;
 
   @BeforeClass
   public static void setup() throws Exception {
@@ -60,7 +61,7 @@ public class TestApplicationHistoryClientService {
     TimelineStore store =
         TestApplicationHistoryManagerOnTimelineStore.createStore(2);
     TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
-    TimelineDataManager dataManager =
+    dataManager =
         new TimelineDataManager(store, aclsManager);
     ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
     ApplicationHistoryManagerOnTimelineStore historyManager =
@@ -101,8 +102,27 @@ public class TestApplicationHistoryClientService {
         clientService.getApplications(request);
     List<ApplicationReport> appReport = response.getApplicationList();
     Assert.assertNotNull(appReport);
-    Assert.assertEquals(appId, appReport.get(0).getApplicationId());
-    Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
+    Assert.assertEquals(appId, appReport.get(1).getApplicationId());
+    Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
+
+    // Create a historyManager, and set the max_apps can be loaded
+    // as 1.
+    Configuration conf = new YarnConfiguration();
+    conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+    ApplicationHistoryManagerOnTimelineStore historyManager2 =
+        new ApplicationHistoryManagerOnTimelineStore(dataManager,
+          new ApplicationACLsManager(conf));
+    historyManager2.init(conf);
+    historyManager2.start();
+    @SuppressWarnings("resource")
+    ApplicationHistoryClientService clientService2 =
+        new ApplicationHistoryClientService(historyManager2);
+    response = clientService2.getApplications(request);
+    appReport = response.getApplicationList();
+    Assert.assertNotNull(appReport);
+    Assert.assertTrue(appReport.size() == 1);
+    // Expected to get the appReport for application with appId1
+    Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index fcdafc2..8672953 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -180,8 +180,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
       Assert.assertEquals("test app type", app.getApplicationType());
       Assert.assertEquals("user1", app.getUser());
       Assert.assertEquals("test queue", app.getQueue());
-      Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
-      Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
+      Assert.assertEquals(Integer.MAX_VALUE + 2L
+          + app.getApplicationId().getId(), app.getStartTime());
+      Assert.assertEquals(Integer.MAX_VALUE + 3L
+          + +app.getApplicationId().getId(), app.getFinishTime());
       Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
       // App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
       // Nobody except admin and owner has access to the details of the app.
@@ -335,7 +337,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
   @Test
   public void testGetApplications() throws Exception {
     Collection<ApplicationReport> apps =
-        historyManager.getAllApplications().values();
+        historyManager.getApplications(Long.MAX_VALUE).values();
     Assert.assertNotNull(apps);
     Assert.assertEquals(SCALE + 1, apps.size());
     ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
@@ -472,12 +474,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
     entity.setOtherInfo(entityInfo);
     TimelineEvent tEvent = new TimelineEvent();
     tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
-    tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+    tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
     entity.addEvent(tEvent);
     tEvent = new TimelineEvent();
     tEvent.setEventType(
         ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
-    tEvent.setTimestamp(Integer.MAX_VALUE + 3L);
+    tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
     Map<String, Object> eventInfo = new HashMap<String, Object>();
     eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
         "test diagnostics info");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index 5df70c4..b0a3f2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.webapp;
 
 import static org.apache.hadoop.yarn.util.StringHelper.join;
 import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
 import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
 
@@ -70,10 +71,14 @@ public class AppsBlock extends HtmlBlock {
         reqAppStates.add(YarnApplicationState.valueOf(stateString.trim()));
       }
     }
-
     callerUGI = getCallerUGI();
     final GetApplicationsRequest request =
         GetApplicationsRequest.newInstance(reqAppStates);
+    String appsNumStr = $(APPS_NUM);
+    if (appsNumStr != null && !appsNumStr.isEmpty()) {
+      long appsNum = Long.parseLong(appsNumStr);
+      request.setLimit(appsNum);
+    }
     if (callerUGI == null) {
       appReports = appBaseProt.getApplications(request).getApplicationList();
     } else {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index 909bf1d..8404719 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -72,13 +72,11 @@ public class WebServices {
       String startedEnd, String finishBegin, String finishEnd,
       Set<String> applicationTypes) {
     UserGroupInformation callerUGI = getUser(req);
-    long num = 0;
-    boolean checkCount = false;
     boolean checkStart = false;
     boolean checkEnd = false;
     boolean checkAppTypes = false;
     boolean checkAppStates = false;
-    long countNum = 0;
+    long countNum = Long.MAX_VALUE;
 
     // set values suitable in case both of begin/end not specified
     long sBegin = 0;
@@ -87,7 +85,6 @@ public class WebServices {
     long fEnd = Long.MAX_VALUE;
 
     if (count != null && !count.isEmpty()) {
-      checkCount = true;
       countNum = Long.parseLong(count);
       if (countNum <= 0) {
         throw new BadRequestException("limit value must be greater then 0");
@@ -148,19 +145,20 @@ public class WebServices {
 
     AppsInfo allApps = new AppsInfo();
     Collection<ApplicationReport> appReports = null;
+    final GetApplicationsRequest request =
+        GetApplicationsRequest.newInstance();
+    request.setLimit(countNum);
     try {
       if (callerUGI == null) {
         // TODO: the request should take the params like what RMWebServices does
         // in YARN-1819.
-        GetApplicationsRequest request = GetApplicationsRequest.newInstance();
         appReports = appBaseProt.getApplications(request).getApplicationList();
       } else {
         appReports = callerUGI.doAs(
             new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
           @Override
           public Collection<ApplicationReport> run() throws Exception {
-            return appBaseProt.getApplications(
-                GetApplicationsRequest.newInstance()).getApplicationList();
+            return appBaseProt.getApplications(request).getApplicationList();
           }
         });
       }
@@ -169,10 +167,6 @@ public class WebServices {
     }
     for (ApplicationReport appReport : appReports) {
 
-      if (checkCount && num == countNum) {
-        break;
-      }
-
       if (checkAppStates &&
           !appStates.contains(StringUtils.toLowerCase(
               appReport.getYarnApplicationState().toString()))) {
@@ -212,7 +206,6 @@ public class WebServices {
       AppInfo app = new AppInfo(appReport);
 
       allApps.add(app);
-      num++;
     }
     return allApps;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index e177215..5a18fb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -183,6 +183,7 @@ selected if this policy is either of `HTTPS_ONLY` or `HTTP_AND_HTTPS`.
 | `yarn.timeline-service.handler-thread-count` | Handler thread count to serve the client RPC requests. Defaults to `10`. |
 | `yarn.timeline-service.client.max-retries` | The maximum number of retries for attempts to publish data to the timeline service.Defaults to `30`. |
 | `yarn.timeline-service.client.retry-interval-ms` | The interval in milliseconds between retries for the timeline service client. Defaults to `1000`. |
+| `yarn.timeline-service.generic-application-history.max-applications` | The max number of applications could be fetched by using REST API or application history protocol and shown in timeline server web ui. Defaults to `10000`. |
 
 
 


[30/50] [abbrv] hadoop git commit: HDFS-7401. Add block info to DFSInputStream' WARN message when it adds node to deadNodes (Contributed by Arshad Mohammad)

Posted by zj...@apache.org.
HDFS-7401. Add block info to DFSInputStream' WARN message when it adds node to deadNodes (Contributed by Arshad Mohammad)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f26da60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f26da60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f26da60

Branch: refs/heads/YARN-2928
Commit: 0f26da602f6f5ae84b4975cb85c2d4f48c4e394b
Parents: fdc6ac8
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri May 29 16:36:19 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt                      | 3 +++
 .../src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java     | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f26da60/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 514076b..23699a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -819,6 +819,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8407. hdfsListDirectory must set errno to 0 on success (Masatake
     Iwasaki via Colin P. McCabe)
 
+    HDFS-7401. Add block info to DFSInputStream' WARN message when it adds
+    node to deadNodes (Arshad Mohammad via vinayakumarb)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f26da60/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index d1e0b9a..8a3f730 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -631,8 +631,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
           fetchBlockAt(target);
         } else {
           connectFailedOnce = true;
-          DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block"
-            + ", add to deadNodes and continue. " + ex, ex);
+          DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block "
+            +targetBlock.getBlock()+ ", add to deadNodes and continue. " + ex, ex);
           // Put chosen node into dead list, continue
           addToDeadNodes(chosenNode);
         }


[28/50] [abbrv] hadoop git commit: HADOOP-12022. fix site -Pdocs -Pdist in hadoop-project-dist; cleanout remaining forrest bits (aw)

Posted by zj...@apache.org.
 HADOOP-12022. fix site -Pdocs -Pdist in hadoop-project-dist; cleanout remaining forrest bits (aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b17498d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b17498d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b17498d6

Branch: refs/heads/YARN-2928
Commit: b17498d6e188a61cdc89b84df4bfef83152d2ecf
Parents: 44f4df5
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 13:23:49 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |     3 +
 hadoop-common-project/hadoop-common/pom.xml     |     1 -
 .../src/main/docs/changes/ChangesFancyStyle.css |   170 -
 .../main/docs/changes/ChangesSimpleStyle.css    |    49 -
 .../src/main/docs/changes/changes2html.pl       |   286 -
 .../src/main/docs/releasenotes.html             | 29099 -----------------
 .../src/main/docs/src/documentation/README.txt  |     7 -
 .../classes/CatalogManager.properties           |    40 -
 .../main/docs/src/documentation/conf/cli.xconf  |   327 -
 .../src/documentation/content/xdocs/index.xml   |    48 -
 .../src/documentation/content/xdocs/site.xml    |   263 -
 .../src/documentation/content/xdocs/tabs.xml    |    37 -
 .../resources/images/architecture.gif           |   Bin 15461 -> 0 bytes
 .../resources/images/common-logo.jpg            |   Bin 5887 -> 0 bytes
 .../resources/images/core-logo.gif              |   Bin 6665 -> 0 bytes
 .../documentation/resources/images/favicon.ico  |   Bin 766 -> 0 bytes
 .../resources/images/hadoop-logo-big.jpg        |   Bin 127869 -> 0 bytes
 .../resources/images/hadoop-logo.jpg            |   Bin 9443 -> 0 bytes
 .../resources/images/hdfsarchitecture.gif       |   Bin 17653 -> 0 bytes
 .../resources/images/hdfsarchitecture.odg       |   Bin 41298 -> 0 bytes
 .../resources/images/hdfsarchitecture.png       |   Bin 40571 -> 0 bytes
 .../resources/images/hdfsdatanodes.gif          |   Bin 16060 -> 0 bytes
 .../resources/images/hdfsdatanodes.odg          |   Bin 37296 -> 0 bytes
 .../resources/images/hdfsdatanodes.png          |   Bin 30012 -> 0 bytes
 .../main/docs/src/documentation/skinconf.xml    |   366 -
 .../hadoop-common/src/main/docs/status.xml      |    75 -
 hadoop-hdfs-project/hadoop-hdfs/pom.xml         |     2 -
 hadoop-project-dist/pom.xml                     |    16 -
 28 files changed, 3 insertions(+), 30786 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index aba9087..71b71a0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -477,6 +477,9 @@ Trunk (Unreleased)
     HADOOP-11983. HADOOP_USER_CLASSPATH_FIRST works the opposite of what it is
     supposed to do (Sangjin Lee via aw)
 
+    HADOOP-12022. fix site -Pdocs -Pdist in hadoop-project-dist; cleanout
+    remaining forrest bits (aw)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 6f9ecf7..3043889 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -488,7 +488,6 @@
             <exclude>CHANGES.txt</exclude>
             <exclude>.idea/**</exclude>
             <exclude>src/main/conf/*</exclude>
-            <exclude>src/main/docs/**</exclude>
             <exclude>dev-support/jdiff/**</exclude>
             <exclude>src/main/native/*</exclude>
             <exclude>src/main/native/config/*</exclude>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesFancyStyle.css
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesFancyStyle.css b/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesFancyStyle.css
deleted file mode 100644
index 5eef241..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesFancyStyle.css
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-/**
- * General
- */
-
-img { border: 0; }
-
-#content table {
-  border: 0;
-  width: 100%;
-}
-/*Hack to get IE to render the table at 100%*/
-* html #content table { margin-left: -3px; }
-
-#content th,
-#content td {
-  margin: 0;
-  padding: 0;
-  vertical-align: top;
-}
-
-.clearboth {
-  clear: both;
-}
-
-.note, .warning, .fixme {
-  border: solid black 1px;
-  margin: 1em 3em;
-}
-
-.note .label {
-  background: #369;
-  color: white;
-  font-weight: bold;
-  padding: 5px 10px;
-}
-.note .content {
-  background: #F0F0FF;
-  color: black;
-  line-height: 120%;
-  font-size: 90%;
-  padding: 5px 10px;
-}
-.warning .label {
-  background: #C00;
-  color: white;
-  font-weight: bold;
-  padding: 5px 10px;
-}
-.warning .content {
-  background: #FFF0F0;
-  color: black;
-  line-height: 120%;
-  font-size: 90%;
-  padding: 5px 10px;
-}
-.fixme .label {
-  background: #C6C600;
-  color: black;
-  font-weight: bold;
-  padding: 5px 10px;
-}
-.fixme .content {
-  padding: 5px 10px;
-}
-
-/**
- * Typography
- */
-
-body {
-  font-family: verdana, "Trebuchet MS", arial, helvetica, sans-serif;
-  font-size: 100%;
-}
-
-#content {
-  font-family: Georgia, Palatino, Times, serif;
-  font-size: 95%;
-}
-#tabs {
-  font-size: 70%;
-}
-#menu {
-  font-size: 80%;
-}
-#footer {
-  font-size: 70%;
-}
-
-h1, h2, h3, h4, h5, h6 {
-  font-family: "Trebuchet MS", verdana, arial, helvetica, sans-serif;
-  font-weight: bold;
-  margin-top: 1em;
-  margin-bottom: .5em;
-}
-
-h1 {
-    margin-top: 0;
-    margin-bottom: 1em;
-  font-size: 1.4em;
-  background-color: 73CAFF
-}
-#content h1 {
-  font-size: 160%;
-  margin-bottom: .5em;
-}
-#menu h1 {
-  margin: 0;
-  padding: 10px;
-  background: #336699;
-  color: white;
-}
-h2 { 
-  font-size: 120%;
-  background-color: 73CAFF
-}
-h3 { font-size: 100%; }
-h4 { font-size: 90%; }
-h5 { font-size: 80%; }
-h6 { font-size: 75%; }
-
-p {
-  line-height: 120%;
-  text-align: left;
-  margin-top: .5em;
-  margin-bottom: 1em;
-}
-
-#content li,
-#content th,
-#content td,
-#content li ul,
-#content li ol{
-  margin-top: .5em;
-  margin-bottom: .5em;
-}
-
-
-#content li li,
-#minitoc-area li{
-  margin-top: 0em;
-  margin-bottom: 0em;
-}
-
-#content .attribution {
-  text-align: right;
-  font-style: italic;
-  font-size: 85%;
-  margin-top: 1em;
-}
-
-.codefrag {
-  font-family: "Courier New", Courier, monospace;
-  font-size: 110%;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesSimpleStyle.css
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesSimpleStyle.css b/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesSimpleStyle.css
deleted file mode 100644
index 407d0f1..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesSimpleStyle.css
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-body {
-  font-family: Courier New, monospace;
-  font-size: 10pt;
-}
-
-h1 {
-  font-family: Courier New, monospace;
-  font-size: 10pt;
-}
-
-h2 {
-  font-family: Courier New, monospace;
-  font-size: 10pt; 
-}
-
-h3 {
-  font-family: Courier New, monospace;
-  font-size: 10pt; 
-}
-
-a:link {
-  color: blue;
-}
-
-a:visited {
-  color: purple; 
-}
-
-li {
-  margin-top: 1em;
-  margin-bottom: 1em;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl b/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
deleted file mode 100755
index 4431e57..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
+++ /dev/null
@@ -1,286 +0,0 @@
-#!/usr/bin/perl
-#
-# Transforms Lucene Java's CHANGES.txt into Changes.html
-#
-# Input is on STDIN, output is to STDOUT
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-use strict;
-use warnings;
-
-my $jira_url_prefix = 'http://issues.apache.org/jira/browse/';
-my $title = undef;
-my $release = undef;
-my $sections = undef;
-my $items = undef;
-my $first_relid = undef;
-my $second_relid = undef;
-my @releases = ();
-
-my @lines = <>;                        # Get all input at once
-
-#
-# Parse input and build hierarchical release structure in @releases
-#
-for (my $line_num = 0 ; $line_num <= $#lines ; ++$line_num) {
-  $_ = $lines[$line_num];
-  next unless (/\S/);                  # Skip blank lines
-
-  unless ($title) {
-    if (/\S/) {
-      s/^\s+//;                        # Trim leading whitespace
-      s/\s+$//;                        # Trim trailing whitespace
-    }
-    $title = $_;
-    next;
-  }
-
-  if (/^(Release)|(Trunk)/) {   # Release headings
-    $release = $_;
-    $sections = [];
-    push @releases, [ $release, $sections ];
-    ($first_relid = lc($release)) =~ s/\s+/_/g   if ($#releases == 0);
-    ($second_relid = lc($release)) =~ s/\s+/_/g  if ($#releases == 1);
-    $items = undef;
-    next;
-  }
-
-  # Section heading: 2 leading spaces, words all capitalized
-  if (/^  ([A-Z]+)\s*/) {
-    my $heading = $_;
-    $items = [];
-    push @$sections, [ $heading, $items ];
-    next;
-  }
-
-  # Handle earlier releases without sections - create a headless section
-  unless ($items) {
-    $items = [];
-    push @$sections, [ undef, $items ];
-  }
-
-  my $type;
-  if (@$items) { # A list item has been encountered in this section before
-    $type = $items->[0];  # 0th position of items array is list type
-  } else {
-    $type = get_list_type($_);
-    push @$items, $type;
-  }
-
-  if ($type eq 'numbered') { # The modern items list style
-    # List item boundary is another numbered item or an unindented line
-    my $line;
-    my $item = $_;
-    $item =~ s/^(\s{0,2}\d+\.\s*)//;       # Trim the leading item number
-    my $leading_ws_width = length($1);
-    $item =~ s/\s+$//;                     # Trim trailing whitespace
-    $item .= "\n";
-
-    while ($line_num < $#lines
-           and ($line = $lines[++$line_num]) !~ /^(?:\s{0,2}\d+\.\s*\S|\S)/) {
-      $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace
-      $line =~ s/\s+$//;                   # Trim trailing whitespace
-      $item .= "$line\n";
-    }
-    $item =~ s/\n+\Z/\n/;                  # Trim trailing blank lines
-    push @$items, $item;
-    --$line_num unless ($line_num == $#lines);
-  } elsif ($type eq 'paragraph') {         # List item boundary is a blank line
-    my $line;
-    my $item = $_;
-    $item =~ s/^(\s+)//;
-    my $leading_ws_width = defined($1) ? length($1) : 0;
-    $item =~ s/\s+$//;                     # Trim trailing whitespace
-    $item .= "\n";
-
-    while ($line_num < $#lines and ($line = $lines[++$line_num]) =~ /\S/) {
-      $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace
-      $line =~ s/\s+$//;                   # Trim trailing whitespace
-      $item .= "$line\n";
-    }
-    push @$items, $item;
-    --$line_num unless ($line_num == $#lines);
-  } else { # $type is one of the bulleted types
-    # List item boundary is another bullet or a blank line
-    my $line;
-    my $item = $_;
-    $item =~ s/^(\s*$type\s*)//;           # Trim the leading bullet
-    my $leading_ws_width = length($1);
-    $item =~ s/\s+$//;                     # Trim trailing whitespace
-    $item .= "\n";
-
-    while ($line_num < $#lines
-           and ($line = $lines[++$line_num]) !~ /^\s*(?:$type|\Z)/) {
-      $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace
-      $line =~ s/\s+$//;                   # Trim trailing whitespace
-      $item .= "$line\n";
-    }
-    push @$items, $item;
-    --$line_num unless ($line_num == $#lines);
-  }
-}
-
-#
-# Print HTML-ified version to STDOUT
-#
-print<<"__HTML_HEADER__";
-<!--
-**********************************************************
-** WARNING: This file is generated from CHANGES.txt by the 
-**          Perl script 'changes2html.pl'.
-**          Do *not* edit this file!
-**********************************************************
-          
-****************************************************************************
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-****************************************************************************
--->
-<html>
-<head>
-  <title>$title</title>
-  <link rel="stylesheet" href="ChangesFancyStyle.css" title="Fancy">
-  <link rel="alternate stylesheet" href="ChangesSimpleStyle.css" title="Simple">
-  <META http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
-  <SCRIPT>
-    function toggleList(e) {
-      element = document.getElementById(e).style;
-      element.display == 'none' ? element.display = 'block' : element.display='none';
-    }
-    function collapse() {
-      for (var i = 0; i < document.getElementsByTagName("ul").length; i++) {
-        var list = document.getElementsByTagName("ul")[i];
-        if (list.id != '$first_relid' && list.id != '$second_relid') {
-          list.style.display = "none";
-        }
-      }
-      for (var i = 0; i < document.getElementsByTagName("ol").length; i++) {
-        document.getElementsByTagName("ol")[i].style.display = "none"; 
-      }
-    }
-    window.onload = collapse;
-  </SCRIPT>
-</head>
-<body>
-
-<a href="http://hadoop.apache.org/core/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Scalable Computing Platform"></a>
-<h1>$title</h1>
-
-__HTML_HEADER__
-
-my $heading;
-my $relcnt = 0;
-my $header = 'h2';
-for my $rel (@releases) {
-  if (++$relcnt == 3) {
-    $header = 'h3';
-    print "<h2><a href=\"javascript:toggleList('older')\">";
-    print "Older Releases";
-    print "</a></h2>\n";
-    print "<ul id=\"older\">\n"
-  }
-      
-  ($release, $sections) = @$rel;
-
-  # The first section heading is undefined for the older sectionless releases
-  my $has_release_sections = $sections->[0][0];
-
-  (my $relid = lc($release)) =~ s/\s+/_/g;
-  print "<$header><a href=\"javascript:toggleList('$relid')\">";
-  print "$release";
-  print "</a></$header>\n";
-  print "<ul id=\"$relid\">\n"
-    if ($has_release_sections);
-
-  for my $section (@$sections) {
-    ($heading, $items) = @$section;
-    (my $sectid = lc($heading)) =~ s/\s+/_/g;
-    my $numItemsStr = $#{$items} > 0 ? "($#{$items})" : "(none)";  
-
-    print "  <li><a href=\"javascript:toggleList('$relid.$sectid')\">",
-          ($heading || ''), "</a>&nbsp;&nbsp;&nbsp;$numItemsStr\n"
-      if ($has_release_sections);
-
-    my $list_type = $items->[0] || '';
-    my $list = ($has_release_sections || $list_type eq 'numbered' ? 'ol' : 'ul');
-    my $listid = $sectid ? "$relid.$sectid" : $relid;
-    print "    <$list id=\"$listid\">\n";
-
-    for my $itemnum (1..$#{$items}) {
-      my $item = $items->[$itemnum];
-      $item =~ s:&:&amp;:g;                            # Escape HTML metachars
-      $item =~ s:<:&lt;:g; 
-      $item =~ s:>:&gt;:g;
-
-      $item =~ s:\s*(\([^)"]+?\))\s*$:<br />$1:;       # Separate attribution
-      $item =~ s:\n{2,}:\n<p/>\n:g;                    # Keep paragraph breaks
-      $item =~ s{(?:${jira_url_prefix})?(HADOOP-\d+)}  # Link to JIRA Common
-                {<a href="${jira_url_prefix}$1">$1</a>}g;
-      $item =~ s{(?:${jira_url_prefix})?(HDFS-\d+)}    # Link to JIRA Hdfs
-                {<a href="${jira_url_prefix}$1">$1</a>}g;
-      $item =~ s{(?:${jira_url_prefix})?(MAPREDUCE-\d+)}  # Link to JIRA MR
-                {<a href="${jira_url_prefix}$1">$1</a>}g;
-      print "      <li>$item</li>\n";
-    }
-    print "    </$list>\n";
-    print "  </li>\n" if ($has_release_sections);
-  }
-  print "</ul>\n" if ($has_release_sections);
-}
-print "</ul>\n" if ($relcnt > 3);
-print "</body>\n</html>\n";
-
-
-#
-# Subroutine: get_list_type
-#
-# Takes one parameter:
-#
-#    - The first line of a sub-section/point
-#
-# Returns one scalar:
-#
-#    - The list type: 'numbered'; or one of the bulleted types '-', or '.' or
-#      'paragraph'.
-#
-sub get_list_type {
-  my $first_list_item_line = shift;
-  my $type = 'paragraph'; # Default to paragraph type
-
-  if ($first_list_item_line =~ /^\s{0,2}\d+\.\s+\S+/) {
-    $type = 'numbered';
-  } elsif ($first_list_item_line =~ /^\s*([-.])\s+\S+/) {
-    $type = $1;
-  }
-  return $type;
-}
-
-1;


[08/50] [abbrv] hadoop git commit: HDFS-8431. hdfs crypto class not found in Windows. Contributed by Anu Engineer.

Posted by zj...@apache.org.
HDFS-8431. hdfs crypto class not found in Windows. Contributed by Anu Engineer.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b39cff2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b39cff2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b39cff2

Branch: refs/heads/YARN-2928
Commit: 2b39cff2e8545ad4756d4ead42e52661be7bfc22
Parents: 81dd9e8
Author: cnauroth <cn...@apache.org>
Authored: Wed May 27 22:54:00 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:53 2015 -0700

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt           | 3 +++
 hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd | 7 ++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b39cff2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 439b53a..47481a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -810,6 +810,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-8268. Port conflict log for data node server is not sufficient
     (Mohammad Shahid Khan via vinayakumarb)
 
+    HDFS-8431. hdfs crypto class not found in Windows.
+    (Anu Engineer via cnauroth)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b39cff2/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
index 7883252..8115349 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
@@ -59,7 +59,7 @@ if "%1" == "--loglevel" (
     )
   )
 
-  set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath
+  set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto
   for %%i in ( %hdfscommands% ) do (
     if %hdfs-command% == %%i set hdfscommand=true
   )
@@ -175,6 +175,10 @@ goto :eof
   set CLASS=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
   goto :eof
 
+:crypto
+  set CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
+  goto :eof
+
 @rem This changes %1, %2 etc. Hence those cannot be used after calling this.
 :make_command_arguments
   if "%1" == "--config" (
@@ -227,6 +231,7 @@ goto :eof
   @echo   lsSnapshottableDir   list all snapshottable dirs owned by the current user
   @echo 						Use -help to see options
   @echo   cacheadmin           configure the HDFS cache
+  @echo   crypto               configure HDFS encryption zones
   @echo   mover                run a utility to move block replicas across storage types
   @echo   storagepolicies      list/get/set block storage policies
   @echo.


[43/50] [abbrv] hadoop git commit: YARN-2900. Application (Attempt and Container) Not Found in AHS results in Internal Server Error (500). Contributed by Zhijie Shen and Mit Desai

Posted by zj...@apache.org.
YARN-2900. Application (Attempt and Container) Not Found in AHS results
in Internal Server Error (500). Contributed by Zhijie Shen and Mit Desai


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a977bed5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a977bed5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a977bed5

Branch: refs/heads/YARN-2928
Commit: a977bed50936c7054f270f93569fa0f53002fe0b
Parents: 8bc36be
Author: Xuan <xg...@apache.org>
Authored: Sun May 31 15:37:05 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:13:00 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../TestApplicationHistoryClientService.java    | 70 +++++++++++++++++++-
 .../webapp/TestAHSWebServices.java              | 63 +++++++++++++++++-
 .../hadoop/yarn/server/webapp/WebServices.java  | 27 +++++---
 4 files changed, 151 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a977bed5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6f5faa9..e770a5b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -694,6 +694,9 @@ Release 2.7.1 - UNRELEASED
     YARN-3686. CapacityScheduler should trim default_node_label_expression. 
     (Sunil G via wangda)
 
+    YARN-2900. Application (Attempt and Container) Not Found in AHS results
+    in Internal Server Error (500). (Zhijie Shen and Mit Desai via xgong)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a977bed5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 764ecb9..d04903c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
@@ -54,12 +57,13 @@ public class TestApplicationHistoryClientService {
 
   private static ApplicationHistoryClientService clientService;
   private static TimelineDataManager dataManager;
+  private final static int MAX_APPS = 2;
 
   @BeforeClass
   public static void setup() throws Exception {
     Configuration conf = new YarnConfiguration();
     TimelineStore store =
-        TestApplicationHistoryManagerOnTimelineStore.createStore(2);
+        TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
     TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
     dataManager =
         new TimelineDataManager(store, aclsManager);
@@ -72,6 +76,70 @@ public class TestApplicationHistoryClientService {
   }
 
   @Test
+  public void testApplicationNotFound() throws IOException, YarnException {
+    ApplicationId appId = null;
+    appId = ApplicationId.newInstance(0, MAX_APPS + 1);
+    GetApplicationReportRequest request =
+        GetApplicationReportRequest.newInstance(appId);
+    try {
+      @SuppressWarnings("unused")
+      GetApplicationReportResponse response =
+          clientService.getApplicationReport(request);
+      Assert.fail("Exception should have been thrown before we reach here.");
+    } catch (ApplicationNotFoundException e) {
+      //This exception is expected.
+      Assert.assertTrue(e.getMessage().contains(
+          "doesn't exist in the timeline store"));
+    } catch (Exception e) {
+      Assert.fail("Undesired exception caught");
+    }
+  }
+
+  @Test
+  public void testApplicationAttemptNotFound() throws IOException, YarnException {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
+    GetApplicationAttemptReportRequest request =
+        GetApplicationAttemptReportRequest.newInstance(appAttemptId);
+    try {
+      @SuppressWarnings("unused")
+      GetApplicationAttemptReportResponse response =
+          clientService.getApplicationAttemptReport(request);
+      Assert.fail("Exception should have been thrown before we reach here.");
+    } catch (ApplicationAttemptNotFoundException e) {
+      //This Exception is expected
+      System.out.println(e.getMessage());
+      Assert.assertTrue(e.getMessage().contains(
+          "doesn't exist in the timeline store"));
+    } catch (Exception e) {
+      Assert.fail("Undesired exception caught");
+    }
+  }
+
+  @Test
+  public void testContainerNotFound() throws IOException, YarnException {
+   ApplicationId appId = ApplicationId.newInstance(0, 1);
+   ApplicationAttemptId appAttemptId =
+       ApplicationAttemptId.newInstance(appId, 1);
+   ContainerId containerId = ContainerId.newContainerId(appAttemptId,
+       MAX_APPS + 1);
+   GetContainerReportRequest request =
+       GetContainerReportRequest.newInstance(containerId);
+   try {
+   @SuppressWarnings("unused")
+   GetContainerReportResponse response =
+       clientService.getContainerReport(request);
+   } catch (ContainerNotFoundException e) {
+     //This exception is expected
+     Assert.assertTrue(e.getMessage().contains(
+         "doesn't exist in the timeline store"));
+   }  catch (Exception e) {
+      Assert.fail("Undesired exception caught");
+   }
+ }
+
+  @Test
   public void testApplicationReport() throws IOException, YarnException {
     ApplicationId appId = null;
     appId = ApplicationId.newInstance(0, 1);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a977bed5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index 613df72..b5ff6a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -82,12 +82,13 @@ public class TestAHSWebServices extends JerseyTestBase {
 
   private static ApplicationHistoryClientService historyClientService;
   private static final String[] USERS = new String[] { "foo" , "bar" };
+  private static final int MAX_APPS = 5;
 
   @BeforeClass
   public static void setupClass() throws Exception {
     Configuration conf = new YarnConfiguration();
     TimelineStore store =
-        TestApplicationHistoryManagerOnTimelineStore.createStore(5);
+        TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
     TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
     TimelineDataManager dataManager =
         new TimelineDataManager(store, aclsManager);
@@ -165,6 +166,66 @@ public class TestAHSWebServices extends JerseyTestBase {
   }
 
   @Test
+  public void testInvalidApp() {
+    ApplicationId appId = ApplicationId.newInstance(0, MAX_APPS + 1);
+    WebResource r = resource();
+    ClientResponse response =
+        r.path("ws").path("v1").path("applicationhistory").path("apps")
+          .path(appId.toString())
+          .queryParam("user.name", USERS[round])
+          .accept(MediaType.APPLICATION_JSON)
+          .get(ClientResponse.class);
+    assertEquals("404 not found expected", Status.NOT_FOUND,
+        response.getClientResponseStatus());
+  }
+
+  @Test
+  public void testInvalidAttempt() {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
+    WebResource r = resource();
+    ClientResponse response =
+        r.path("ws").path("v1").path("applicationhistory").path("apps")
+          .path(appId.toString()).path("appattempts")
+          .path(appAttemptId.toString())
+          .queryParam("user.name", USERS[round])
+          .accept(MediaType.APPLICATION_JSON)
+          .get(ClientResponse.class);
+    if (round == 1) {
+      assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
+      return;
+    }
+    assertEquals("404 not found expected", Status.NOT_FOUND,
+            response.getClientResponseStatus());
+  }
+
+  @Test
+  public void testInvalidContainer() throws Exception {
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId containerId = ContainerId.newContainerId(appAttemptId,
+        MAX_APPS + 1);
+    WebResource r = resource();
+    ClientResponse response =
+        r.path("ws").path("v1").path("applicationhistory").path("apps")
+          .path(appId.toString()).path("appattempts")
+          .path(appAttemptId.toString()).path("containers")
+          .path(containerId.toString())
+          .queryParam("user.name", USERS[round])
+          .accept(MediaType.APPLICATION_JSON)
+          .get(ClientResponse.class);
+    if (round == 1) {
+      assertEquals(
+          Status.FORBIDDEN, response.getClientResponseStatus());
+      return;
+    }
+    assertEquals("404 not found expected", Status.NOT_FOUND,
+            response.getClientResponseStatus());
+  }
+
+  @Test
   public void testInvalidUri() throws JSONException, Exception {
     WebResource r = resource();
     String responseStr = "";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a977bed5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index 8404719..3064145 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -47,6 +47,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
 import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@@ -477,17 +480,21 @@ public class WebServices {
 
   private static void rewrapAndThrowException(Exception e) {
     if (e instanceof UndeclaredThrowableException) {
-      if (e.getCause() instanceof AuthorizationException) {
-        throw new ForbiddenException(e.getCause());
-      } else {
-        throw new WebApplicationException(e.getCause());
-      }
+      rewrapAndThrowThrowable(e.getCause());
     } else {
-      if (e instanceof AuthorizationException) {
-        throw new ForbiddenException(e);
-      } else {
-        throw new WebApplicationException(e);
-      }
+      rewrapAndThrowThrowable(e);
+    }
+  }
+
+  private static void rewrapAndThrowThrowable(Throwable t) {
+    if (t instanceof AuthorizationException) {
+      throw new ForbiddenException(t);
+    } else if (t instanceof ApplicationNotFoundException ||
+        t instanceof ApplicationAttemptNotFoundException ||
+        t instanceof ContainerNotFoundException) {
+      throw new NotFoundException(t);
+    } else {
+      throw new WebApplicationException(t);
     }
   }
 


[26/50] [abbrv] hadoop git commit: HADOOP-12022. fix site -Pdocs -Pdist in hadoop-project-dist; cleanout remaining forrest bits (aw)

Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/README.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/README.txt b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/README.txt
deleted file mode 100644
index 9bc261b..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/README.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-This is the base documentation directory.
-
-skinconf.xml     # This file customizes Forrest for your project. In it, you
-                 # tell forrest the project name, logo, copyright info, etc
-
-sitemap.xmap     # Optional. This sitemap is consulted before all core sitemaps.
-                 # See http://forrest.apache.org/docs/project-sitemap.html

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/classes/CatalogManager.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/classes/CatalogManager.properties b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/classes/CatalogManager.properties
deleted file mode 100644
index b9cb584..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/classes/CatalogManager.properties
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#=======================================================================
-# CatalogManager.properties
-#
-# This is the default properties file for Apache Forrest.
-# This facilitates local configuration of application-specific catalogs.
-#
-# See the Apache Forrest documentation:
-# http://forrest.apache.org/docs/your-project.html
-# http://forrest.apache.org/docs/validation.html
-
-# verbosity ... level of messages for status/debug
-# See forrest/src/core/context/WEB-INF/cocoon.xconf
-
-# catalogs ... list of additional catalogs to load
-#  (Note that Apache Forrest will automatically load its own default catalog
-#  from src/core/context/resources/schema/catalog.xcat)
-# use full pathnames
-# pathname separator is always semi-colon (;) regardless of operating system
-# directory separator is always slash (/) regardless of operating system
-#
-#catalogs=/home/me/forrest/my-site/src/documentation/resources/schema/catalog.xcat
-catalogs=
-

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/conf/cli.xconf
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/conf/cli.xconf b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/conf/cli.xconf
deleted file mode 100644
index 5c6e245..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/conf/cli.xconf
+++ /dev/null
@@ -1,327 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<!--+
-    |  This is the Apache Cocoon command line configuration file.
-    |  Here you give the command line interface details of where
-    |  to find various aspects of your Cocoon installation.
-    |
-    |  If you wish, you can also use this file to specify the URIs
-    |  that you wish to generate.
-    |
-    |  The current configuration information in this file is for
-    |  building the Cocoon documentation. Therefore, all links here
-    |  are relative to the build context dir, which, in the build.xml
-    |  file, is set to ${build.context}
-    |
-    |  Options:
-    |    verbose:            increase amount of information presented
-    |                        to standard output (default: false)
-    |    follow-links:       whether linked pages should also be
-    |                        generated (default: true)
-    |    precompile-only:    precompile sitemaps and XSP pages, but
-    |                        do not generate any pages (default: false)
-    |    confirm-extensions: check the mime type for the generated page
-    |                        and adjust filename and links extensions
-    |                        to match the mime type
-    |                        (e.g. text/html->.html)
-    |
-    |  Note: Whilst using an xconf file to configure the Cocoon
-    |        Command Line gives access to more features, the use of
-    |        command line parameters is more stable, as there are
-    |        currently plans to improve the xconf format to allow
-    |        greater flexibility. If you require a stable and
-    |        consistent method for accessing the CLI, it is recommended
-    |        that you use the command line parameters to configure
-    |        the CLI. See documentation at:
-    |        http://cocoon.apache.org/2.1/userdocs/offline/
-    |        http://wiki.apache.org/cocoon/CommandLine
-    |
-    +-->
-
-<cocoon verbose="true"
-        follow-links="true"
-        precompile-only="false"
-        confirm-extensions="false">
-
-   <!--+
-       |  The context directory is usually the webapp directory
-       |  containing the sitemap.xmap file.
-       |
-       |  The config file is the cocoon.xconf file.
-       |
-       |  The work directory is used by Cocoon to store temporary
-       |  files and cache files.
-       |
-       |  The destination directory is where generated pages will
-       |  be written (assuming the 'simple' mapper is used, see
-       |  below)
-       +-->
-   <context-dir>.</context-dir>
-   <config-file>WEB-INF/cocoon.xconf</config-file>
-   <work-dir>../tmp/cocoon-work</work-dir>
-   <dest-dir>../site</dest-dir>
-
-   <!--+
-       |  A checksum file can be used to store checksums for pages
-       |  as they are generated. When the site is next generated,
-       |  files will not be written if their checksum has not changed.
-       |  This means that it will be easier to detect which files
-       |  need to be uploaded to a server, using the timestamp.
-       |
-       |  The default path is relative to the core webapp directory.
-       |  An asolute path can be used.
-       +-->
-   <!--   <checksums-uri>build/work/checksums</checksums-uri>-->
-
-   <!--+
-       | Broken link reporting options:
-       |   Report into a text file, one link per line:
-       |     <broken-links type="text" report="filename"/>
-       |   Report into an XML file:
-       |     <broken-links type="xml" report="filename"/>
-       |   Ignore broken links (default):
-       |     <broken-links type="none"/>
-       |
-       |   Two attributes to this node specify whether a page should
-       |   be generated when an error has occured. 'generate' specifies
-       |   whether a page should be generated (default: true) and
-       |   extension specifies an extension that should be appended
-       |   to the generated page's filename (default: none)
-       |
-       |   Using this, a quick scan through the destination directory
-       |   will show broken links, by their filename extension.
-       +-->
-   <broken-links type="xml"
-                 file="../brokenlinks.xml"
-                 generate="false"
-                 extension=".error"
-                 show-referrers="true"/>
-
-   <!--+
-       |  Load classes at startup. This is necessary for generating
-       |  from sites that use SQL databases and JDBC.
-       |  The <load-class> element can be repeated if multiple classes
-       |  are needed.
-       +-->
-   <!--
-   <load-class>org.firebirdsql.jdbc.Driver</load-class>
-   -->
-
-   <!--+
-       |  Configures logging.
-       |  The 'log-kit' parameter specifies the location of the log kit
-       |  configuration file (usually called logkit.xconf.
-       |
-       |  Logger specifies the logging category (for all logging prior
-       |  to other Cocoon logging categories taking over)
-       |
-       |  Available log levels are:
-       |    DEBUG:        prints all level of log messages.
-       |    INFO:         prints all level of log messages except DEBUG
-       |                  ones.
-       |    WARN:         prints all level of log messages except DEBUG
-       |                  and INFO ones.
-       |    ERROR:        prints all level of log messages except DEBUG,
-       |                  INFO and WARN ones.
-       |    FATAL_ERROR:  prints only log messages of this level
-       +-->
-   <!-- <logging log-kit="WEB-INF/logkit.xconf" logger="cli" level="ERROR" /> -->
-
-   <!--+
-       |  Specifies the filename to be appended to URIs that
-       |  refer to a directory (i.e. end with a forward slash).
-       +-->
-   <default-filename>index.html</default-filename>
-
-   <!--+
-       |  Specifies a user agent string to the sitemap when
-       |  generating the site.
-       |
-       |  A generic term for a web browser is "user agent". Any
-       |  user agent, when connecting to a web server, will provide
-       |  a string to identify itself (e.g. as Internet Explorer or
-       |  Mozilla). It is possible to have Cocoon serve different
-       |  content depending upon the user agent string provided by
-       |  the browser. If your site does this, then you may want to
-       |  use this <user-agent> entry to provide a 'fake' user agent
-       |  to Cocoon, so that it generates the correct version of your
-       |  site.
-       |
-       |  For most sites, this can be ignored.
-       +-->
-   <!--
-   <user-agent>Cocoon Command Line Environment 2.1</user-agent>
-   -->
-
-   <!--+
-       |  Specifies an accept string to the sitemap when generating
-       |  the site.
-       |  User agents can specify to an HTTP server what types of content
-       |  (by mime-type) they are able to receive. E.g. a browser may be
-       |  able to handle jpegs, but not pngs. The HTTP accept header
-       |  allows the server to take the browser's capabilities into account,
-       |  and only send back content that it can handle.
-       |
-       |  For most sites, this can be ignored.
-       +-->
-
-   <accept>*/*</accept>
-
-   <!--+
-       | Specifies which URIs should be included or excluded, according
-       | to wildcard patterns.
-       |
-       | These includes/excludes are only relevant when you are following
-       | links. A link URI must match an include pattern (if one is given)
-       | and not match an exclude pattern, if it is to be followed by
-       | Cocoon. It can be useful, for example, where there are links in
-       | your site to pages that are not generated by Cocoon, such as
-       | references to api-documentation.
-       |
-       | By default, all URIs are included. If both include and exclude
-       | patterns are specified, a URI is first checked against the
-       | include patterns, and then against the exclude patterns.
-       |
-       | Multiple patterns can be given, using muliple include or exclude
-       | nodes.
-       |
-       | The order of the elements is not significant, as only the first
-       | successful match of each category is used.
-       |
-       | Currently, only the complete source URI can be matched (including
-       | any URI prefix). Future plans include destination URI matching
-       | and regexp matching. If you have requirements for these, contact
-       | dev@cocoon.apache.org.
-       +-->
-
-   <exclude pattern="**/"/>
-   <exclude pattern="api/**"/>
-   <exclude pattern="jdiff/**"/>
-   <exclude pattern="changes.html"/>
-   <exclude pattern="releasenotes.html"/>
-
-<!--
-  This is a workaround for FOR-284 "link rewriting broken when
-  linking to xml source views which contain site: links".
-  See the explanation there and in declare-broken-site-links.xsl
--->
-   <exclude pattern="site:**"/>
-   <exclude pattern="ext:**"/>
-   <exclude pattern="lm:**"/>
-   <exclude pattern="**/site:**"/>
-   <exclude pattern="**/ext:**"/>
-   <exclude pattern="**/lm:**"/>
-
-   <!-- Exclude tokens used in URLs to ASF mirrors (interpreted by a CGI) -->
-   <exclude pattern="[preferred]/**"/>
-   <exclude pattern="[location]"/>
-
-   <!--   <include-links extension=".html"/>-->
-
-   <!--+
-       |  <uri> nodes specify the URIs that should be generated, and
-       |  where required, what should be done with the generated pages.
-       |  They describe the way the URI of the generated file is created
-       |  from the source page's URI. There are three ways that a generated
-       |  file URI can be created: append, replace and insert.
-       |
-       |  The "type" attribute specifies one of (append|replace|insert):
-       |
-       |  append:
-       |  Append the generated page's URI to the end of the source URI:
-       |
-       |   <uri type="append" src-prefix="documents/" src="index.html"
-       |   dest="build/dest/"/>
-       |
-       |  This means that
-       |   (1) the "documents/index.html" page is generated
-       |   (2) the file will be written to "build/dest/documents/index.html"
-       |
-       |  replace:
-       |  Completely ignore the generated page's URI - just
-       |  use the destination URI:
-       |
-       |   <uri type="replace" src-prefix="documents/" src="index.html"
-       |   dest="build/dest/docs.html"/>
-       |
-       |  This means that
-       |   (1) the "documents/index.html" page is generated
-       |   (2) the result is written to "build/dest/docs.html"
-       |   (3) this works only for "single" pages - and not when links
-       |       are followed
-       |
-       |  insert:
-       |  Insert generated page's URI into the destination
-       |  URI at the point marked with a * (example uses fictional
-       |  zip protocol)
-       |
-       |   <uri type="insert" src-prefix="documents/" src="index.html"
-       |   dest="zip://*.zip/page.html"/>
-       |
-       |  This means that
-       |   (1)
-       |
-       |  In any of these scenarios, if the dest attribute is omitted,
-       |  the value provided globally using the <dest-dir> node will
-       |  be used instead.
-       +-->
-   <!--
-   <uri type="replace"
-        src-prefix="samples/"
-        src="hello-world/hello.html"
-        dest="build/dest/hello-world.html"/>
-   -->
-
-   <!--+
-       | <uri> nodes can be grouped together in a <uris> node. This
-       | enables a group of URIs to share properties. The following
-       | properties can be set for a group of URIs:
-       |   * follow-links:       should pages be crawled for links
-       |   * confirm-extensions: should file extensions be checked
-       |                         for the correct mime type
-       |   * src-prefix:         all source URIs should be
-       |                         pre-pended with this prefix before
-       |                         generation. The prefix is not
-       |                         included when calculating the
-       |                         destination URI
-       |   * dest:               the base destination URI to be
-       |                         shared by all pages in this group
-       |   * type:               the method to be used to calculate
-       |                         the destination URI. See above
-       |                         section on <uri> node for details.
-       |
-       | Each <uris> node can have a name attribute. When a name
-       | attribute has been specified, the -n switch on the command
-       | line can be used to tell Cocoon to only process the URIs
-       | within this URI group. When no -n switch is given, all
-       | <uris> nodes are processed. Thus, one xconf file can be
-       | used to manage multiple sites.
-       +-->
-   <!--
-   <uris name="mirrors" follow-links="false">
-     <uri type="append" src="mirrors.html"/>
-   </uris>
-   -->
-
-   <!--+
-       |  File containing URIs (plain text, one per line).
-       +-->
-   <!--
-   <uri-file>uris.txt</uri-file>
-   -->
-</cocoon>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/index.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/index.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/index.xml
deleted file mode 100644
index e0df9c9..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/index.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
-
-<document>
-  
-  <header>
-    <title>Overview</title>
-  </header>
-  
-  <body>
-  <p>
-The Hadoop Common Documentation describes the common utilities and libraries that support the other Hadoop subprojects.  
-  </p>
-  <p>
-The Hadoop Common Documentation also includes the information you need to get started using Hadoop. 
-Begin with the Hadoop <a href="single_node_setup.html">Single Node Setup</a> which shows you how to set up a single-node Hadoop installation. 
-Then move on to the Hadoop <a href="cluster_setup.html">Cluster Setup</a> to learn how to set up a multi-node Hadoop installation. 
-</p>
- <p>
-   Cluster environments commonly work in tandem with MapReduce applications and distributed file systems. 
-   For information about MapReduce see the 
- <a href="http://hadoop.apache.org/mapreduce/docs/current/index.html">MapReduce Documentation</a>.
-   For information about the Hadoop Distributed File System (HDFS) see the 
- <a href="http://hadoop.apache.org/hdfs/docs/current/index.html">HDFS Documentation</a>.
-  </p>  
-<p>
-If you have more questions, you can ask on the <a href="ext:lists">Hadoop Common Mailing Lists</a> or browse the <a href="ext:archive">Mailing List Archives</a>.
-</p>
-  </body>
-  
-</document>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml
deleted file mode 100644
index f9ce3be..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml
+++ /dev/null
@@ -1,263 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-<!--
-Forrest site.xml
-
-This file contains an outline of the site's information content.  It is used to:
-- Generate the website menus (though these can be overridden - see docs)
-- Provide semantic, location-independent aliases for internal 'site:' URIs, eg
-<link href="site:changes"> links to changes.html (or ../changes.html if in
-  subdir).
-- Provide aliases for external URLs in the external-refs section.  Eg, <link
-  href="ext:cocoon"> links to http://xml.apache.org/cocoon/
-
-See http://forrest.apache.org/docs/linking.html for more info.
--->
-
-<site label="Hadoop" href="" xmlns="http://apache.org/forrest/linkmap/1.0">
-  
-   <docs label="Getting Started"> 
-		<overview   				label="Overview" 					href="index.html" />
-		<quickstart 				label="Single Node Setup"      href="single_node_setup.html" />
-		<setup     					label="Cluster Setup"      		href="cluster_setup.html" />
-  </docs>	
-		
- <docs label="Guides">
-		<deployment					label="Deployment Layout" href="deployment_layout.html" />
-		<commands_manual 				label="Hadoop Commands"  href="commands_manual.html" />
-		<fsshell				        label="File System Shell"               href="file_system_shell.html" />
-		<SLA					 	label="Service Level Authorization" 	href="service_level_auth.html"/>
-		<native_lib    				label="Native Libraries" 					href="native_libraries.html" />
-                <superusers                      label="Superusers Acting On Behalf Of Other Users"     href="Superusers.html"/>
-    <http_authentication label="Authentication for Hadoop HTTP web-consoles" href="HttpAuthentication.html"/>
-   </docs>
-
-   <docs label="Miscellaneous"> 
-		<api       	label="API Docs"           href="ext:api/index" />
-		<jdiff     	label="API Changes"      href="ext:jdiff/changes" />
-		<wiki      	label="Wiki"               	href="ext:wiki" />
-		<faq       	label="FAQ"                	href="ext:faq" />
-		<relnotes  label="Release Notes" 	href="ext:relnotes" />
-		<changes	label="Change Log"       href="ext:changes" />
-   </docs> 
-   
-  <external-refs>
-    <site      href="http://hadoop.apache.org/common/"/>
-    <lists     href="http://hadoop.apache.org/common/mailing_lists.html"/>
-    <archive   href="http://mail-archives.apache.org/mod_mbox/hadoop-common-commits/"/>
-    <releases  href="http://hadoop.apache.org/common/releases.html">
-      <download href="#Download" />
-    </releases>
-    <jira  href="http://hadoop.apache.org/common/issue_tracking.html"/>
-    <wiki  href="http://wiki.apache.org/hadoop/Common" />
-    <faq  href="http://wiki.apache.org/hadoop/Common/FAQ" />
-    
-    <core-default href="http://hadoop.apache.org/common/docs/current/core-default.html" />
-    <hdfs-default href="http://hadoop.apache.org/hdfs/docs/current/hdfs-default.html" />
-    <mapred-default href="http://hadoop.apache.org/mapreduce/docs/current/mapred-default.html" />
-    
-    <mapred-queues href="http://hadoop.apache.org/mapreduce/docs/current/mapred_queues.xml" />
-    <capacity-scheduler href="http://hadoop.apache.org/mapreduce/docs/current/capacity_scheduler.html">
-        <MemoryBasedTaskScheduling href="#Scheduling+Tasks+Considering+Memory+Requirements" />
-    </capacity-scheduler>
-    <mapred-tutorial href="http://hadoop.apache.org/mapreduce/docs/current/mapred_tutorial.html" >
-        <JobAuthorization href="#Job+Authorization" />
-        <ConfiguringMemoryRequirements href="#Configuring+Memory+Requirements+For+A+Job" />
-    </mapred-tutorial>
-    <streaming href="http://hadoop.apache.org/mapreduce/docs/current/streaming.html" />
-    <distcp href="http://hadoop.apache.org/mapreduce/docs/current/distcp.html" />
-    <hadoop-archives href="http://hadoop.apache.org/mapreduce/docs/current/hadoop_archives.html" />
-    
-    <zlib      href="http://www.zlib.net/" />
-    <gzip      href="http://www.gzip.org/" />
-    <bzip      href="http://www.bzip.org/" />
-    <osx       href="http://www.apple.com/macosx" />
-    
-    <relnotes href="releasenotes.html" />
-    <changes href="changes.html" />
-    <jdiff href="jdiff/">
-      <changes href="changes.html" />
-    </jdiff>
-    <api href="api/">
-      <index href="index.html" />
-      <org href="org/">
-        <apache href="apache/">
-          <hadoop href="hadoop/">
-            <conf href="conf/">
-              <configuration href="Configuration.html">
-                <final_parameters href="#FinalParams" />
-                <get href="#get(java.lang.String, java.lang.String)" />
-                <set href="#set(java.lang.String, java.lang.String)" />
-              </configuration>
-            </conf>
-            <filecache href="filecache/">
-              <distributedcache href="DistributedCache.html">
-                <addarchivetoclasspath href="#addArchiveToClassPath(org.apache.hadoop.fs.Path,%20org.apache.hadoop.conf.Configuration)" />
-                <addfiletoclasspath href="#addFileToClassPath(org.apache.hadoop.fs.Path,%20org.apache.hadoop.conf.Configuration)" />
-                <addcachefile href="#addCacheFile(java.net.URI,%20org.apache.hadoop.conf.Configuration)" />
-                <addcachearchive href="#addCacheArchive(java.net.URI,%20org.apache.hadoop.conf.Configuration)" />
-                <setcachefiles href="#setCacheFiles(java.net.URI[],%20org.apache.hadoop.conf.Configuration)" />
-                <setcachearchives href="#setCacheArchives(java.net.URI[],%20org.apache.hadoop.conf.Configuration)" />
-                <createsymlink href="#createSymlink(org.apache.hadoop.conf.Configuration)" />
-              </distributedcache>  
-            </filecache>
-            <fs href="fs/">
-              <filesystem href="FileSystem.html" />
-            </fs>
-            <io href="io/">
-              <closeable href="Closeable.html">
-                <close href="#close()" />
-              </closeable>
-              <sequencefile href="SequenceFile.html" />
-              <sequencefilecompressiontype href="SequenceFile.CompressionType.html">
-                <none href="#NONE" />
-                <record href="#RECORD" />
-                <block href="#BLOCK" />
-              </sequencefilecompressiontype>
-              <writable href="Writable.html" />
-              <writablecomparable href="WritableComparable.html" />
-              <compress href="compress/">
-                <compressioncodec href="CompressionCodec.html" />
-              </compress>
-            </io>
-            <mapred href="mapred/">
-              <clusterstatus href="ClusterStatus.html" />
-              <counters href="Counters.html" />
-              <fileinputformat href="FileInputFormat.html">
-                 <setinputpaths href="#setInputPaths(org.apache.hadoop.mapred.JobConf,%20org.apache.hadoop.fs.Path[])" />
-                 <addinputpath href="#addInputPath(org.apache.hadoop.mapred.JobConf,%20org.apache.hadoop.fs.Path)" />
-                 <setinputpathstring href="#setInputPaths(org.apache.hadoop.mapred.JobConf,%20java.lang.String)" />
-                 <addinputpathstring href="#addInputPath(org.apache.hadoop.mapred.JobConf,%20java.lang.String)" />
-              </fileinputformat>
-              <fileoutputformat href="FileOutputFormat.html">
-                <getoutputpath href="#getOutputPath(org.apache.hadoop.mapred.JobConf)" />
-                <getworkoutputpath href="#getWorkOutputPath(org.apache.hadoop.mapred.JobConf)" />
-                <setoutputpath href="#setOutputPath(org.apache.hadoop.mapred.JobConf,%20org.apache.hadoop.fs.Path)" />
-                <setcompressoutput href="#setCompressOutput(org.apache.hadoop.mapred.JobConf,%20boolean)" />
-                <setoutputcompressorclass href="#setOutputCompressorClass(org.apache.hadoop.mapred.JobConf,%20java.lang.Class)" />
-              </fileoutputformat>
-              <filesplit href="FileSplit.html" />
-              <inputformat href="InputFormat.html" />
-              <inputsplit href="InputSplit.html" />
-              <isolationrunner href="IsolationRunner.html" />
-              <jobclient href="JobClient.html">
-                <runjob href="#runJob(org.apache.hadoop.mapred.JobConf)" />
-                <submitjob href="#submitJob(org.apache.hadoop.mapred.JobConf)" />
-              </jobclient>
-              <jobconf href="JobConf.html">
-                <setnummaptasks href="#setNumMapTasks(int)" />
-                <setnumreducetasks href="#setNumReduceTasks(int)" />
-                <setoutputkeycomparatorclass href="#setOutputKeyComparatorClass(java.lang.Class)" />
-                <setoutputvaluegroupingcomparator href="#setOutputValueGroupingComparator(java.lang.Class)" />
-                <setcombinerclass href="#setCombinerClass(java.lang.Class)" />
-                <setmapdebugscript href="#setMapDebugScript(java.lang.String)" />
-                <setreducedebugscript href="#setReduceDebugScript(java.lang.String)" />
-                <setmapspeculativeexecution href="#setMapSpeculativeExecution(boolean)" />
-                <setreducespeculativeexecution href="#setReduceSpeculativeExecution(boolean)" />
-                <setmaxmapattempts href="#setMaxMapAttempts(int)" />
-                <setmaxreduceattempts href="#setMaxReduceAttempts(int)" />
-                <setmaxmaptaskfailurespercent href="#setMaxMapTaskFailuresPercent(int)" />
-                <setmaxreducetaskfailurespercent href="#setMaxReduceTaskFailuresPercent(int)" />
-                <setjobendnotificationuri href="#setJobEndNotificationURI(java.lang.String)" />
-                <setcompressmapoutput href="#setCompressMapOutput(boolean)" />
-                <setmapoutputcompressorclass href="#setMapOutputCompressorClass(java.lang.Class)" />
-                <setprofileenabled href="#setProfileEnabled(boolean)" />
-                <setprofiletaskrange href="#setProfileTaskRange(boolean,%20java.lang.String)" />
-                <setprofileparams href="#setProfileParams(java.lang.String)" />
-                <setnumtaskstoexecuteperjvm href="#setNumTasksToExecutePerJvm(int)" />
-                <setqueuename href="#setQueueName(java.lang.String)" />
-                <getjoblocaldir href="#getJobLocalDir()" />
-                <getjar href="#getJar()" />
-              </jobconf>
-              <jobconfigurable href="JobConfigurable.html">
-                <configure href="#configure(org.apache.hadoop.mapred.JobConf)" />
-              </jobconfigurable>
-              <jobcontrol href="jobcontrol/">
-                <package-summary href="package-summary.html" />
-              </jobcontrol>
-              <mapper href="Mapper.html">
-                <map href="#map(K1, V1, org.apache.hadoop.mapred.OutputCollector, org.apache.hadoop.mapred.Reporter)" />
-              </mapper>
-              <outputcollector href="OutputCollector.html">
-                <collect href="#collect(K, V)" />
-              </outputcollector>
-              <outputcommitter href="OutputCommitter.html" />
-              <outputformat href="OutputFormat.html" />
-              <outputlogfilter href="OutputLogFilter.html" />
-              <sequencefileoutputformat href="SequenceFileOutputFormat.html">
-                <setoutputcompressiontype href="#setOutputCompressionType(org.apache.hadoop.mapred.JobConf,%20org.apache.hadoop.io.SequenceFile.CompressionType)" />
-              </sequencefileoutputformat>
-              <partitioner href="Partitioner.html" />
-              <recordreader href="RecordReader.html" />
-              <recordwriter href="RecordWriter.html" />
-              <reducer href="Reducer.html">
-                <reduce href="#reduce(K2, java.util.Iterator, org.apache.hadoop.mapred.OutputCollector, org.apache.hadoop.mapred.Reporter)" />
-              </reducer>
-              <reporter href="Reporter.html">
-                <incrcounterEnum href="#incrCounter(java.lang.Enum, long)" />
-                <incrcounterString href="#incrCounter(java.lang.String, java.lang.String, long amount)" />
-              </reporter>
-              <runningjob href="RunningJob.html" />
-              <skipbadrecords href="SkipBadRecords.html">
-                <setmappermaxskiprecords href="#setMapperMaxSkipRecords(org.apache.hadoop.conf.Configuration, long)"/>
-                <setreducermaxskipgroups href="#setReducerMaxSkipGroups(org.apache.hadoop.conf.Configuration, long)"/>
-                <setattemptsTostartskipping href="#setAttemptsToStartSkipping(org.apache.hadoop.conf.Configuration, int)"/>
-                <setskipoutputpath href="#setSkipOutputPath(org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path)"/>
-                <counter_map_processed_records href="#COUNTER_MAP_PROCESSED_RECORDS"/>
-                <counter_reduce_processed_groups href="#COUNTER_REDUCE_PROCESSED_GROUPS"/>
-              </skipbadrecords>
-              <textinputformat href="TextInputFormat.html" />
-              <textoutputformat href="TextOutputFormat.html" />
-              <lib href="lib/">
-                <package-summary href="package-summary.html" />
-                <hashpartitioner href="HashPartitioner.html" />
-                <keyfieldbasedpartitioner href="KeyFieldBasedPartitioner.html" />
-                <keyfieldbasedcomparator href="KeyFieldBasedComparator.html" />
-                <lazyoutputformat href="LazyOutputFormat.html" />
-                <aggregate href="aggregate/">
-                  <package-summary href="package-summary.html" />
-                </aggregate>
-              </lib>
-              <pipes href="pipes/">
-                <package-summary href="package-summary.html" />
-              </pipes>
-            </mapred>
-            <net href="net/">
-              <dnstoswitchmapping href="DNSToSwitchMapping.html">
-              <resolve href="#resolve(java.util.List)" />
-              </dnstoswitchmapping>
-            </net>
-            <streaming href="streaming/">
-              <package-summary href="package-summary.html" />
-            </streaming>
-            <util href="util/">
-              <genericoptionsparser href="GenericOptionsParser.html" />
-              <progress href="Progress.html" />
-              <tool href="Tool.html" />
-              <toolrunner href="ToolRunner.html">
-                <run href="#run(org.apache.hadoop.util.Tool, java.lang.String[])" />
-              </toolrunner>
-            </util>
-          </hadoop>
-        </apache>
-      </org>
-    </api>
-  </external-refs>
- 
-</site>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/tabs.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/tabs.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/tabs.xml
deleted file mode 100644
index ffd556c..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/tabs.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-<!DOCTYPE tabs PUBLIC "-//APACHE//DTD Cocoon Documentation Tab V1.0//EN" 
-          "http://forrest.apache.org/dtd/tab-cocoon-v10.dtd">
-
-<tabs software="Hadoop"
-      title="Hadoop"
-      copyright="The Apache Software Foundation"
-      xmlns:xlink="http://www.w3.org/1999/xlink">
-
-  <!-- The rules are:
-    @dir will always have /index.html added.
-    @href is not modified unless it is root-relative and obviously specifies a
-    directory (ends in '/'), in which case /index.html will be added
-  -->
-
-  <tab label="Project" href="http://hadoop.apache.org/common/" />
-  <tab label="Wiki" href="http://wiki.apache.org/hadoop" />
-  <tab label="Common 0.24 Documentation" dir="" />  
-  
-</tabs>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/architecture.gif
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/architecture.gif b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/architecture.gif
deleted file mode 100644
index 8d84a23..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/architecture.gif and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/common-logo.jpg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/common-logo.jpg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/common-logo.jpg
deleted file mode 100644
index 8ba52cd..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/common-logo.jpg and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/core-logo.gif
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/core-logo.gif b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/core-logo.gif
deleted file mode 100644
index 57879bb..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/core-logo.gif and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/favicon.ico
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/favicon.ico b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/favicon.ico
deleted file mode 100644
index 161bcf7..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/favicon.ico and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo-big.jpg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo-big.jpg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo-big.jpg
deleted file mode 100644
index 0c6996c..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo-big.jpg and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo.jpg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo.jpg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo.jpg
deleted file mode 100644
index 809525d..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo.jpg and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.gif
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.gif b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.gif
deleted file mode 100644
index 7bbe150..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.gif and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.odg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.odg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.odg
deleted file mode 100644
index dae62bf..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.odg and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.png
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.png b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.png
deleted file mode 100644
index 70e434c..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.png and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.gif
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.gif b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.gif
deleted file mode 100644
index b01f56f..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.gif and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.odg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.odg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.odg
deleted file mode 100644
index e5d9227..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.odg and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.png
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.png b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.png
deleted file mode 100644
index 40ea2ec..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.png and /dev/null differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/skinconf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/skinconf.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/skinconf.xml
deleted file mode 100644
index 3c93d7f..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/skinconf.xml
+++ /dev/null
@@ -1,366 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-<!--
-Skin configuration file. This file contains details of your project,
-which will be used to configure the chosen Forrest skin.
--->
-
-<!DOCTYPE skinconfig PUBLIC "-//APACHE//DTD Skin Configuration V0.6-3//EN" "http://forrest.apache.org/dtd/skinconfig-v06-3.dtd">
-<skinconfig>
-  <!-- To enable lucene search add provider="lucene" (default is google).
-    Add box-location="alt" to move the search box to an alternate location
-    (if the skin supports it) and box-location="all" to show it in all
-    available locations on the page.  Remove the <search> element to show
-    no search box. @domain will enable sitesearch for the specific domain with google.
-    In other words google will search the @domain for the query string.
-
-  -->
-  <search name="Lucene" domain="hadoop.apache.org" provider="google"/>
-
-  <!-- Disable the print link? If enabled, invalid HTML 4.0.1 -->
-  <disable-print-link>true</disable-print-link>  
-  <!-- Disable the PDF link? -->
-  <disable-pdf-link>false</disable-pdf-link>
-  <!-- Disable the POD link? -->
-  <disable-pod-link>true</disable-pod-link>
-  <!-- Disable the Text link? FIXME: NOT YET IMPLEMENETED. -->
-  <disable-txt-link>true</disable-txt-link>
-  <!-- Disable the xml source link? -->
-  <!-- The xml source link makes it possible to access the xml rendition
-    of the source frim the html page, and to have it generated statically.
-    This can be used to enable other sites and services to reuse the
-    xml format for their uses. Keep this disabled if you don't want other
-    sites to easily reuse your pages.-->
-  <disable-xml-link>true</disable-xml-link>
-
-  <!-- Disable navigation icons on all external links? -->
-  <disable-external-link-image>true</disable-external-link-image>
-
-  <!-- Disable w3c compliance links? 
-    Use e.g. align="center" to move the compliance links logos to 
-    an alternate location default is left.
-    (if the skin supports it) -->
-  <disable-compliance-links>true</disable-compliance-links>
-
-  <!-- Render mailto: links unrecognisable by spam harvesters? -->
-  <obfuscate-mail-links>false</obfuscate-mail-links>
-
-  <!-- Disable the javascript facility to change the font size -->
-  <disable-font-script>true</disable-font-script>
-
-  <!-- project logo -->
-  <project-name>Hadoop</project-name>
-  <project-description>Scalable Computing Platform</project-description>
-  <project-url>http://hadoop.apache.org/core/</project-url>
-  <project-logo>images/common-logo.jpg</project-logo>
-
-  <!-- group logo -->
-  <group-name>Hadoop</group-name>
-  <group-description>Apache Hadoop</group-description>
-  <group-url>http://hadoop.apache.org/</group-url>
-  <group-logo>images/hadoop-logo.jpg</group-logo>
-
-  <!-- optional host logo (e.g. sourceforge logo)
-       default skin: renders it at the bottom-left corner -->
-  <host-url></host-url>
-  <host-logo></host-logo>
-
-  <!-- relative url of a favicon file, normally favicon.ico -->
-  <favicon-url>images/favicon.ico</favicon-url>
-
-  <!-- The following are used to construct a copyright statement -->
-  <year>2009</year>
-  <vendor>The Apache Software Foundation.</vendor>
-  <copyright-link>http://www.apache.org/licenses/</copyright-link>
-
-  <!-- Some skins use this to form a 'breadcrumb trail' of links.
-    Use location="alt" to move the trail to an alternate location
-    (if the skin supports it).
-	  Omit the location attribute to display the trail in the default location.
-	  Use location="none" to not display the trail (if the skin supports it).
-    For some skins just set the attributes to blank.
-  -->
-  <trail>
-    <link1 name="Apache" href="http://www.apache.org/"/>
-    <link2 name="Hadoop" href="http://hadoop.apache.org/"/>
-    <link3 name="Common" href="http://hadoop.apache.org/common/"/>
-  </trail>
-
-  <!-- Configure the TOC, i.e. the Table of Contents.
-  @max-depth
-   how many "section" levels need to be included in the
-   generated Table of Contents (TOC). 
-  @min-sections
-   Minimum required to create a TOC.
-  @location ("page","menu","page,menu", "none")
-   Where to show the TOC.
-  -->
-  <toc max-depth="2" min-sections="1" location="page"/>
-
-  <!-- Heading types can be clean|underlined|boxed  -->
-  <headings type="clean"/>
-  
-  <!-- The optional feedback element will be used to construct a
-    feedback link in the footer with the page pathname appended:
-    <a href="@href">{@to}</a>
-  <feedback to="webmaster@foo.com"
-    href="mailto:webmaster@foo.com?subject=Feedback&#160;" >
-    Send feedback about the website to:
-  </feedback>
-    -->
-  <!--
-    extra-css - here you can define custom css-elements that are 
-    a. overriding the fallback elements or 
-    b. adding the css definition from new elements that you may have 
-       used in your documentation.
-    -->
-  <extra-css>
-    <!--Example of b. 
-        To define the css definition of a new element that you may have used
-        in the class attribute of a <p> node. 
-        e.g. <p class="quote"/>
-    -->
-    p.quote {
-      margin-left: 2em;
-      padding: .5em;
-      background-color: #f0f0f0;
-      font-family: monospace;
-    }
-    
-    <!--Headers -->
-	#content h1 {
-	  margin-bottom: .5em;
-	  font-size: 185%; color: black;
-	  font-family: arial;
-	}  
-    h2, .h3 { font-size: 175%; color: black; font-family: arial; }
-	h3, .h4 { font-size: 135%; color: black; font-family: arial; margin-bottom: 0.5em; }
-	h4, .h5 { font-size: 125%; color: black;  font-style: italic; font-weight: bold; font-family: arial; }
-	h5, h6 { font-size: 110%; color: #363636; font-weight: bold; } 
-   
-   <!--Code Background -->
-    pre.code {
-      margin-left: 0em;
-      padding: 0.5em;
-      background-color: rgb(241,239,231);
-      font-family: monospace;
-    }   
-    
-  </extra-css>
-
-  <colors>
-  <!-- These values are used for the generated CSS files. -->
-
-  <!-- Krysalis -->
-<!--
-    <color name="header"    value="#FFFFFF"/>
-
-    <color name="tab-selected" value="#a5b6c6" link="#000000" vlink="#000000" hlink="#000000"/>
-    <color name="tab-unselected" value="#F7F7F7"  link="#000000" vlink="#000000" hlink="#000000"/>
-    <color name="subtab-selected" value="#a5b6c6"  link="#000000" vlink="#000000" hlink="#000000"/>
-    <color name="subtab-unselected" value="#a5b6c6"  link="#000000" vlink="#000000" hlink="#000000"/>
-
-    <color name="heading" value="#a5b6c6"/>
-    <color name="subheading" value="#CFDCED"/>
-        
-    <color name="navstrip" value="#CFDCED" font="#000000" link="#000000" vlink="#000000" hlink="#000000"/>
-    <color name="toolbox" value="#a5b6c6"/>
-    <color name="border" value="#a5b6c6"/>
-        
-    <color name="menu" value="#F7F7F7" link="#000000" vlink="#000000" hlink="#000000"/>    
-    <color name="dialog" value="#F7F7F7"/>
-            
-    <color name="body"    value="#ffffff" link="#0F3660" vlink="#009999" hlink="#000066"/>
-    
-    <color name="table" value="#a5b6c6"/>    
-    <color name="table-cell" value="#ffffff"/>    
-    <color name="highlight" value="#ffff00"/>
-    <color name="fixme" value="#cc6600"/>
-    <color name="note" value="#006699"/>
-    <color name="warning" value="#990000"/>
-    <color name="code" value="#a5b6c6"/>
-        
-    <color name="footer" value="#a5b6c6"/>
--->
-  
-  <!-- Forrest -->
-<!--
-    <color name="header"    value="#294563"/>
-
-    <color name="tab-selected" value="#4a6d8c" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
-    <color name="tab-unselected" value="#b5c7e7" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
-    <color name="subtab-selected" value="#4a6d8c" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
-    <color name="subtab-unselected" value="#4a6d8c" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
-
-    <color name="heading" value="#294563"/>
-    <color name="subheading" value="#4a6d8c"/>
-        
-    <color name="navstrip" value="#cedfef" font="#0F3660" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
-    <color name="toolbox" value="#4a6d8c"/>
-    <color name="border" value="#294563"/>
-    
-    <color name="menu" value="#4a6d8c" font="#cedfef" link="#ffffff" vlink="#ffffff" hlink="#ffcf00"/>    
-    <color name="dialog" value="#4a6d8c"/>
-            
-    <color name="body" value="#ffffff"  link="#0F3660" vlink="#009999" hlink="#000066"/>
-    
-    <color name="table" value="#7099C5"/>    
-    <color name="table-cell" value="#f0f0ff"/>    
-    <color name="highlight" value="#ffff00"/>
-    <color name="fixme" value="#cc6600"/>
-    <color name="note" value="#006699"/>
-    <color name="warning" value="#990000"/>
-    <color name="code" value="#CFDCED"/>
-        
-    <color name="footer" value="#cedfef"/>
--->
-
-  <!-- Collabnet --> 
-<!--
-    <color name="header"    value="#003366"/>
-
-    <color name="tab-selected" value="#dddddd" link="#555555" vlink="#555555" hlink="#555555"/>
-    <color name="tab-unselected" value="#999999" link="#ffffff" vlink="#ffffff" hlink="#ffffff"/>
-    <color name="subtab-selected" value="#cccccc" link="#000000" vlink="#000000" hlink="#000000"/>
-    <color name="subtab-unselected" value="#cccccc" link="#555555" vlink="#555555" hlink="#555555"/>
-
-    <color name="heading" value="#003366"/>
-    <color name="subheading" value="#888888"/>
-    
-    <color name="navstrip" value="#dddddd" font="#555555"/>
-    <color name="toolbox" value="#dddddd" font="#555555"/>
-    <color name="border" value="#999999"/>
-    
-    <color name="menu" value="#ffffff"/>    
-    <color name="dialog" value="#eeeeee"/>
-            
-    <color name="body"      value="#ffffff"/>
-    
-    <color name="table" value="#ccc"/>    
-    <color name="table-cell" value="#ffffff"/>   
-    <color name="highlight" value="#ffff00"/>
-    <color name="fixme" value="#cc6600"/>
-    <color name="note" value="#006699"/>
-    <color name="warning" value="#990000"/>
-    <color name="code" value="#003366"/>
-        
-    <color name="footer" value="#ffffff"/>
--->
- <!-- Lenya using pelt-->
-<!--
-    <color name="header" value="#ffffff"/>
-
-    <color name="tab-selected" value="#4C6C8F" link="#ffffff" vlink="#ffffff" hlink="#ffffff"/>
-    <color name="tab-unselected" value="#E5E4D9" link="#000000" vlink="#000000" hlink="#000000"/>
-    <color name="subtab-selected" value="#000000" link="#000000" vlink="#000000" hlink="#000000"/>
-    <color name="subtab-unselected" value="#E5E4D9" link="#000000" vlink="#000000" hlink="#000000"/>
-
-    <color name="heading" value="#E5E4D9"/>
-    <color name="subheading" value="#000000"/>
-    <color name="published" value="#4C6C8F" font="#FFFFFF"/>
-    <color name="feedback" value="#4C6C8F" font="#FFFFFF" align="center"/>
-    <color name="navstrip" value="#E5E4D9" font="#000000"/>
-
-    <color name="toolbox" value="#CFDCED" font="#000000"/>
-
-    <color name="border" value="#999999"/>
-    <color name="menu" value="#4C6C8F" font="#ffffff" link="#ffffff" vlink="#ffffff" hlink="#ffffff" current="#FFCC33" />    
-    <color name="menuheading" value="#cfdced" font="#000000" />
-    <color name="searchbox" value="#E5E4D9" font="#000000"/>
-    
-    <color name="dialog" value="#CFDCED"/>
-    <color name="body" value="#ffffff" />            
-    
-    <color name="table" value="#ccc"/>    
-    <color name="table-cell" value="#ffffff"/>   
-    <color name="highlight" value="#ffff00"/>
-    <color name="fixme" value="#cc6600"/>
-    <color name="note" value="#006699"/>
-    <color name="warning" value="#990000"/>
-    <color name="code" value="#003366"/>
-        
-    <color name="footer" value="#E5E4D9"/>
--->
-  </colors>
- 
-  <!-- Settings specific to PDF output. -->
-  <pdf>
-    <!-- 
-       Supported page sizes are a0, a1, a2, a3, a4, a5, executive,
-       folio, legal, ledger, letter, quarto, tabloid (default letter).
-       Supported page orientations are portrait, landscape (default
-       portrait).
-       Supported text alignments are left, right, justify (default left).
-    -->
-    <page size="letter" orientation="portrait" text-align="left"/>
-
-    <!--
-       Margins can be specified for top, bottom, inner, and outer
-       edges. If double-sided="false", the inner edge is always left
-       and the outer is always right. If double-sided="true", the
-       inner edge will be left on odd pages, right on even pages,
-       the outer edge vice versa.
-       Specified below are the default settings.
-    -->
-    <margins double-sided="false">
-      <top>1in</top>
-      <bottom>1in</bottom>
-      <inner>1.25in</inner>
-      <outer>1in</outer>
-    </margins>
-
-    <!--
-      Print the URL text next to all links going outside the file
-    -->
-    <show-external-urls>false</show-external-urls>
-
-    <!--
-      Disable the copyright footer on each page of the PDF.
-      A footer is composed for each page. By default, a "credit" with role=pdf
-      will be used, as explained below. Otherwise a copyright statement
-      will be generated. This latter can be disabled.
-    -->
-    <disable-copyright-footer>false</disable-copyright-footer>
-  </pdf>
-
-  <!-- Credits are typically rendered as a set of small clickable
-    images in the page footer.
-    Use box-location="alt" to move the credit to an alternate location
-    (if the skin supports it). 
-  -->
-  <credits>
-    <credit box-location="alt">
-      <name>Built with Apache Forrest</name>
-      <url>http://forrest.apache.org/</url>
-      <image>images/built-with-forrest-button.png</image>
-      <width>88</width>
-      <height>31</height>
-    </credit>
-    <!-- A credit with @role="pdf" will be used to compose a footer
-     for each page in the PDF, using either "name" or "url" or both.
-    -->
-    <!--
-    <credit role="pdf">
-      <name>Built with Apache Forrest</name>
-      <url>http://forrest.apache.org/</url>
-    </credit>
-    -->
-  </credits>
-
-</skinconfig>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/status.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/status.xml b/hadoop-common-project/hadoop-common/src/main/docs/status.xml
deleted file mode 100644
index a820726..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/status.xml
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<status>
-
-  <developers>
-    <person name="Joe Bloggs"      email="joe@joescompany.org"      id="JB" />
-    <!-- Add more people here -->
-  </developers>
-
-  <changes>
-    <!-- Add new releases here -->
-    <release version="0.1" date="unreleased">
-      <!-- Some action types have associated images. By default, images are
-      defined for 'add', 'fix', 'remove', 'update' and 'hack'. If you add
-      src/documentation/resources/images/<foo>.jpg images, these will
-      automatically be used for entries of type <foo>. -->
-
-      <action dev="JB" type="add" context="admin">
-        Initial Import
-      </action>
-      <!-- Sample action:
-      <action dev="JB" type="fix" due-to="Joe Contributor"
-          due-to-email="joec@apache.org" fixes-bug="123">
-          Fixed a bug in the Foo class.
-        </action>
-        -->
-    </release>
-  </changes>
-
-  <todo>
-    <actions priority="high">
-      <action context="docs" dev="JB">
-        Customize this template project with your project's details.  This
-        TODO list is generated from 'status.xml'.
-      </action>
-      <action context="docs" dev="JB">
-        Add lots of content.  XML content goes in
-        <code>src/documentation/content/xdocs</code>, or wherever the
-        <code>${project.xdocs-dir}</code> property (set in
-        <code>forrest.properties</code>) points.
-      </action>
-      <action context="feedback" dev="JB">
-        Mail <link
-          href="mailto:forrest-dev@xml.apache.org">forrest-dev@xml.apache.org</link>
-        with feedback.
-      </action>
-    </actions>
-    <!-- Add todo items. @context is an arbitrary string. Eg:
-    <actions priority="high">
-      <action context="code" dev="SN">
-      </action>
-    </actions>
-    <actions priority="medium">
-      <action context="docs" dev="open">
-      </action>
-    </actions>
-    -->
-  </todo>
-
-</status>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 34338d1..3eeadb6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -373,7 +373,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
             <exclude>.gitattributes</exclude>
             <exclude>.idea/**</exclude>
             <exclude>src/main/conf/*</exclude>
-            <exclude>src/main/docs/**</exclude>
             <exclude>dev-support/findbugsExcludeFile.xml</exclude>
             <exclude>dev-support/checkstyle*</exclude>
             <exclude>dev-support/jdiff/**</exclude>
@@ -390,7 +389,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
             <exclude>src/main/native/util/tree.h</exclude>
             <exclude>src/test/aop/org/apache/hadoop/hdfs/server/datanode/DataXceiverAspects.aj</exclude>
             <exclude>src/main/webapps/datanode/robots.txt</exclude>
-            <exclude>src/main/docs/releasenotes.html</exclude>
             <exclude>src/contrib/**</exclude>
             <exclude>src/site/resources/images/*</exclude>
             <exclude>src/main/webapps/static/bootstrap-3.0.2/**</exclude>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index c26aa9a..e712910 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -246,28 +246,12 @@
 
                     <mkdir dir="${project.build.directory}/docs-src"/>
 
-                    <copy todir="${project.build.directory}/docs-src">
-                      <fileset dir="${basedir}/src/main/docs"/>
-                    </copy>
-
                     <!-- Docs -->
-                    <copy file="${project.build.directory}/docs-src/releasenotes.html"
-                          todir="${project.build.directory}/site"/>
                     <style basedir="${basedir}/src/main/resources"
                            destdir="${project.build.directory}/site"
                            includes="core-default.xml"
                            style="${basedir}/src/main/xsl/configuration.xsl"/>
 
-                    <!-- Convert 'CHANGES.txt' to 'changes.html" -->
-                    <exec executable="perl" input="${basedir}/../CHANGES.txt"
-                          output="${project.build.directory}/site/changes.html"
-                          failonerror="true">
-                      <arg value="${project.build.directory}/docs-src/changes/changes2html.pl"/>
-                    </exec>
-                    <copy todir="${project.build.directory}/site">
-                      <fileset dir="${project.build.directory}/docs-src/changes" includes="*.css"/>
-                    </copy>
-
                     <!-- Jdiff -->
                     <mkdir dir="${project.build.directory}/site/jdiff/xml"/>
 


[17/50] [abbrv] hadoop git commit: HADOOP-11983. HADOOP_USER_CLASSPATH_FIRST works the opposite of what it is supposed to do (Sangjin Lee via aw)

Posted by zj...@apache.org.
HADOOP-11983. HADOOP_USER_CLASSPATH_FIRST works the opposite of what it is supposed to do (Sangjin Lee via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b914ccaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b914ccaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b914ccaf

Branch: refs/heads/YARN-2928
Commit: b914ccafc9647f47e1e667fc10d051d9f7f31e1e
Parents: f47c0e9
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 11:29:58 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt                  | 3 +++
 .../hadoop-common/src/main/bin/hadoop-functions.sh               | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b914ccaf/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 71b3388..69f5aa7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -474,6 +474,9 @@ Trunk (Unreleased)
 
     HADOOP-11406. xargs -P is not portable (Kengo Seki via aw)
 
+    HADOOP-11983. HADOOP_USER_CLASSPATH_FIRST works the opposite of what it is
+    supposed to do (Sangjin Lee via aw)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b914ccaf/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
old mode 100644
new mode 100755
index 5556f2f..be65d7f
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -761,11 +761,11 @@ function hadoop_add_to_classpath_userpath
     if [[ -z "${HADOOP_USE_CLIENT_CLASSLOADER}" ]]; then
       if [[ -z "${HADOOP_USER_CLASSPATH_FIRST}" ]]; then
         for ((i=j; i>=0; i--)); do
-          hadoop_add_classpath "${array[$i]}" before
+          hadoop_add_classpath "${array[$i]}" after
         done
       else
         for ((i=0; i<=j; i++)); do
-          hadoop_add_classpath "${array[$i]}" after
+          hadoop_add_classpath "${array[$i]}" before
         done
       fi
     fi


[24/50] [abbrv] hadoop git commit: HADOOP-11959. WASB should configure client side socket timeout in storage client blob request options. Contributed by Ivan Mitic.

Posted by zj...@apache.org.
HADOOP-11959. WASB should configure client side socket timeout in storage client blob request options. Contributed by Ivan Mitic.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44f4df57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44f4df57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44f4df57

Branch: refs/heads/YARN-2928
Commit: 44f4df57bfe7a078894cc4c05b07c0e3a292412f
Parents: 4b4aeba
Author: cnauroth <cn...@apache.org>
Authored: Thu May 28 12:31:06 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt        |  3 +++
 hadoop-project/pom.xml                                 |  2 +-
 .../hadoop/fs/azure/AzureNativeFileSystemStore.java    | 13 ++-----------
 .../org/apache/hadoop/fs/azure/StorageInterface.java   |  6 +++---
 .../apache/hadoop/fs/azure/StorageInterfaceImpl.java   |  4 ++--
 .../apache/hadoop/fs/azure/MockStorageInterface.java   |  4 ++--
 .../fs/azure/TestAzureFileSystemErrorConditions.java   |  1 +
 .../apache/hadoop/fs/azure/TestBlobDataValidation.java |  1 +
 8 files changed, 15 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6aa224c..aba9087 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -791,6 +791,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-11930. test-patch in offline mode should tell maven to be in
     offline mode (Sean Busbey via aw)
 
+    HADOOP-11959. WASB should configure client side socket timeout in storage
+    client blob request options. (Ivan Mitic via cnauroth)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 94e1f5e..a5d4ed8 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -942,7 +942,7 @@
       <dependency>
         <groupId>com.microsoft.azure</groupId>
         <artifactId>azure-storage</artifactId>
-        <version>2.0.0</version>
+        <version>2.2.0</version>
      </dependency>
 
      <dependency>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 3267d8b..69bda06 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -2434,15 +2434,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
       //
       CloudBlobWrapper dstBlob = getBlobReference(dstKey);
 
-      // TODO: Remove at the time when we move to Azure Java SDK 1.2+.
-      // This is the workaround provided by Azure Java SDK team to
-      // mitigate the issue with un-encoded x-ms-copy-source HTTP
-      // request header. Azure sdk version before 1.2+ does not encode this
-      // header what causes all URIs that have special (category "other")
-      // characters in the URI not to work with startCopyFromBlob when
-      // specified as source (requests fail with HTTP 403).
-      URI srcUri = new URI(srcBlob.getUri().toASCIIString());
-
       // Rename the source blob to the destination blob by copying it to
       // the destination blob then deleting it.
       //
@@ -2451,7 +2442,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
       // a more intensive exponential retry policy when the cluster is getting 
       // throttled.
       try {
-        dstBlob.startCopyFromBlob(srcUri, null, getInstrumentedContext());
+        dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext());
       } catch (StorageException se) {
         if (se.getErrorCode().equals(
 		  StorageErrorCode.SERVER_BUSY.toString())) {
@@ -2475,7 +2466,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
           options.setRetryPolicyFactory(new RetryExponentialRetry(
             copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff, 
 			copyBlobMaxRetries));
-          dstBlob.startCopyFromBlob(srcUri, options, getInstrumentedContext());
+          dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext());
         } else {
           throw se;
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
index e89151d..ce5f749 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
@@ -381,8 +381,8 @@ abstract class StorageInterface {
      * Copies an existing blob's contents, properties, and metadata to this instance of the <code>CloudBlob</code>
      * class, using the specified operation context.
      *
-     * @param source
-     *            A <code>java.net.URI</code> The URI of a source blob.
+     * @param sourceBlob
+     *            A <code>CloudBlob</code> object that represents the source blob to copy.
      * @param options
      *            A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
      *            <code>null</code> will use the default request options from the associated service client (
@@ -397,7 +397,7 @@ abstract class StorageInterface {
      * @throws URISyntaxException
      *
      */
-    public abstract void startCopyFromBlob(URI source,
+    public abstract void startCopyFromBlob(CloudBlobWrapper sourceBlob,
         BlobRequestOptions options, OperationContext opContext)
         throws StorageException, URISyntaxException;
     

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
index 90d4d88..382ff66 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
@@ -393,10 +393,10 @@ class StorageInterfaceImpl extends StorageInterface {
     }
 
     @Override
-    public void startCopyFromBlob(URI source, BlobRequestOptions options,
+    public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
         OperationContext opContext)
             throws StorageException, URISyntaxException {
-      getBlob().startCopyFromBlob(source,
+      getBlob().startCopyFromBlob(((CloudBlobWrapperImpl)sourceBlob).blob,
           null, null, options, opContext);
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
index cde0e38..9f84f4b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
@@ -429,9 +429,9 @@ public class MockStorageInterface extends StorageInterface {
     }
 
     @Override
-    public void startCopyFromBlob(URI source, BlobRequestOptions options,
+    public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
         OperationContext opContext) throws StorageException, URISyntaxException {
-      backingStore.copy(convertUriToDecodedString(source), convertUriToDecodedString(uri));
+      backingStore.copy(convertUriToDecodedString(sourceBlob.getUri()), convertUriToDecodedString(uri));
       //TODO: set the backingStore.properties.CopyState and
       //      update azureNativeFileSystemStore.waitForCopyToComplete
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
index ace57dc..810bcf7 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
@@ -205,6 +205,7 @@ public class TestAzureFileSystemErrorConditions {
         @Override
         public boolean isTargetConnection(HttpURLConnection connection) {
           return connection.getRequestMethod().equals("PUT")
+              && connection.getURL().getQuery() != null
               && connection.getURL().getQuery().contains("blocklist");
         }
       });

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
index 9237ade..c40b7b5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
@@ -191,6 +191,7 @@ public class TestBlobDataValidation {
 
     private static boolean isPutBlock(HttpURLConnection connection) {
       return connection.getRequestMethod().equals("PUT")
+          && connection.getURL().getQuery() != null
           && connection.getURL().getQuery().contains("blockid");
     }
 


[14/50] [abbrv] hadoop git commit: HADOOP-12035. shellcheck plugin displays a wrong version potentially (Kengo Seki via aw)

Posted by zj...@apache.org.
HADOOP-12035. shellcheck plugin displays a wrong version potentially (Kengo Seki via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3eb698cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3eb698cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3eb698cb

Branch: refs/heads/YARN-2928
Commit: 3eb698cb45863fd6e9899fc76983a3832a032ec3
Parents: 7dc7948
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 09:11:08 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700

----------------------------------------------------------------------
 dev-support/test-patch.d/shellcheck.sh          | 2 +-
 hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3eb698cb/dev-support/test-patch.d/shellcheck.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
index 5f38b6a..9277ea5 100755
--- a/dev-support/test-patch.d/shellcheck.sh
+++ b/dev-support/test-patch.d/shellcheck.sh
@@ -72,7 +72,7 @@ function shellcheck_preapply
   start_clock
 
   # shellcheck disable=SC2016
-  SHELLCHECK_VERSION=$(shellcheck --version | ${GREP} version: | ${AWK} '{print $NF}')
+  SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
 
   echo "Running shellcheck against all identifiable shell scripts"
   pushd "${BASEDIR}" >/dev/null

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3eb698cb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 48e9f7d..1009d1a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -765,6 +765,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12004. test-patch breaks with reexec in certain situations (Sean
     Busbey via aw)
 
+    HADOOP-12035. shellcheck plugin displays a wrong version potentially
+    (Kengo Seki via aw)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES


[35/50] [abbrv] hadoop git commit: HADOOP-11885. hadoop-dist dist-layout-stitching.sh does not work with dash. (wang)

Posted by zj...@apache.org.
HADOOP-11885. hadoop-dist dist-layout-stitching.sh does not work with dash. (wang)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/547b9dd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/547b9dd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/547b9dd5

Branch: refs/heads/YARN-2928
Commit: 547b9dd5a3aabe50818b1372e1cf6a9742478068
Parents: 58b91c9
Author: Andrew Wang <an...@cloudera.com>
Authored: Fri May 29 14:54:18 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:58 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt                  | 3 +++
 hadoop-common-project/hadoop-common/pom.xml                      | 2 +-
 hadoop-common-project/hadoop-kms/pom.xml                         | 4 ++--
 hadoop-dist/pom.xml                                              | 4 ++--
 hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml                   | 4 ++--
 hadoop-hdfs-project/hadoop-hdfs/pom.xml                          | 2 +-
 .../hadoop-mapreduce-client-nativetask/pom.xml                   | 2 +-
 hadoop-mapreduce-project/pom.xml                                 | 4 ++--
 hadoop-project-dist/pom.xml                                      | 4 ++--
 .../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml    | 2 +-
 hadoop-yarn-project/pom.xml                                      | 2 +-
 pom.xml                                                          | 2 ++
 12 files changed, 20 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index bc74696..e5d2e39 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -643,6 +643,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-11242. Record the time of calling in tracing span of
     IPC server. (Masatake Iwasaki via aajisaka)
 
+    HADOOP-11885. hadoop-dist dist-layout-stitching.sh does not work with dash.
+    (wang)
+
   BUG FIXES
 
     HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 3043889..c3a306b 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -666,7 +666,7 @@
                 <goals><goal>run</goal></goals>
                 <configuration>
                   <target>
-                    <exec executable="sh" failonerror="true" dir="${project.build.directory}/native">
+                    <exec executable="${shell-executable}" failonerror="true" dir="${project.build.directory}/native">
                       <arg value="-c"/>
                       <arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_bulk_crc32"/>
                       <env key="SKIPTESTS" value="${skipTests}"/>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index c33b33e..ad2062b 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -383,7 +383,7 @@
                       cd "${project.build.directory}/tomcat.exp"
                       gzip -cd ../../downloads/apache-tomcat-${tomcat.version}.tar.gz | tar xf -
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}"
+                    <exec executable="${shell-executable}" dir="${project.build.directory}"
                           failonerror="true">
                       <arg line="./tomcat-untar.sh"/>
                     </exec>
@@ -426,7 +426,7 @@
                       cd "${project.build.directory}"
                       tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}"
+                    <exec executable="${shell-executable}" dir="${project.build.directory}"
                           failonerror="true">
                       <arg line="./dist-maketar.sh"/>
                     </exec>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index f894c01..f1644a8 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -191,7 +191,7 @@
                       echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}"
                       echo
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                    <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
                       <arg line="./dist-layout-stitching.sh"/>
                     </exec>
                   </target>
@@ -224,7 +224,7 @@
                       echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz"
                       echo
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                    <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
                       <arg line="./dist-tar-stitching.sh"/>
                     </exec>
                   </target>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 520e30f..4457597 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -517,7 +517,7 @@
                       cd "${project.build.directory}/tomcat.exp"
                       gzip -cd ../../downloads/apache-tomcat-${tomcat.version}.tar.gz | tar xf -
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                    <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
                       <arg line="./tomcat-untar.sh"/>
                     </exec>
 
@@ -557,7 +557,7 @@
                       cd "${project.build.directory}"
                       tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                    <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
                       <arg line="./dist-maketar.sh"/>
                     </exec>
                   </target>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 3eeadb6..f90644c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -674,7 +674,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
                 <phase>test-compile</phase>
                 <configuration>
                   <target>
-                    <exec executable="sh">
+                    <exec executable="${shell-executable}">
                       <arg value="-c"/>
                       <arg value="for i in {1..${testsThreadCount}}; do mkdir -p ${test.build.data}/$i; mkdir -p ${hadoop.tmp.dir}/$i; done"/>
                     </exec>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 3b0ee48..a5532f1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -197,7 +197,7 @@
                 <goals><goal>run</goal></goals>
                 <configuration>
                   <target>
-                    <exec executable="sh" failonerror="true" dir="${project.build.directory}/native/test">
+                    <exec executable="${shell-executable}" failonerror="true" dir="${project.build.directory}/native/test">
                       <arg value="-c"/>
                       <arg value="[ x$SKIPTESTS = xtrue ] || sh test.sh"/>
                       <env key="SKIPTESTS" value="${skipTests}"/>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-mapreduce-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 088a3ba..0cbd928 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -170,7 +170,7 @@
                   cd "${project.build.directory}"
                   tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                 </echo>
-                <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
                   <arg line="./dist-maketar.sh"/>
                 </exec>
               </target>
@@ -254,7 +254,7 @@
                         cp -R $${LIB_DIR}/lib* $${TARGET_DIR}
                       fi
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                    <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
                       <arg line="./dist-copynativelibs.sh"/>
                     </exec>
                   </target>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index e712910..8df266f 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -361,7 +361,7 @@
                         fi
                       fi
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                    <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
                       <arg line="./dist-copynativelibs.sh"/>
                     </exec>
                   </target>
@@ -380,7 +380,7 @@
                       cd "${project.build.directory}"
                       tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                     </echo>
-                    <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                    <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
                       <arg line="./dist-maketar.sh"/>
                     </exec>
                   </target>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 8e135de..babed8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -203,7 +203,7 @@
                 <phase>test</phase>
                 <configuration>
                   <target>
-                    <exec executable="sh" failonerror="true" dir="${project.build.directory}/native">
+                    <exec executable="${shell-executable}" failonerror="true" dir="${project.build.directory}/native">
                       <arg value="-c"/>
                       <arg value="[ x$SKIPTESTS = xtrue ] || test-container-executor"/>
                       <env key="SKIPTESTS" value="${skipTests}"/>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 736508b..e21e0b8 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -96,7 +96,7 @@
                   cd "${project.build.directory}"
                   tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
                 </echo>
-                <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+                <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
                   <arg line="./dist-maketar.sh"/>
                 </exec>
               </target>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8394324f..4e6f7a1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -105,6 +105,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
     <maven-checkstyle-plugin.version>2.15</maven-checkstyle-plugin.version>
     <checkstyle.version>6.6</checkstyle.version>
+
+    <shell-executable>bash</shell-executable>
   </properties>
 
   <modules>


[41/50] [abbrv] hadoop git commit: YARN-3740. Fixed the typo in the configuration name: APPLICATION_HISTORY_PREFIX_MAX_APPS. Contributed by Xuan Gong.

Posted by zj...@apache.org.
YARN-3740. Fixed the typo in the configuration name: APPLICATION_HISTORY_PREFIX_MAX_APPS. Contributed by Xuan Gong.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0c491be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0c491be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0c491be

Branch: refs/heads/YARN-2928
Commit: d0c491bee6c5bc72f3cb06c01c8142349d666eb4
Parents: 312030a
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri May 29 18:16:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:59 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                                  | 3 +++
 .../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++--
 .../ApplicationHistoryManagerOnTimelineStore.java                | 4 ++--
 .../TestApplicationHistoryClientService.java                     | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0c491be/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 906d9d4..339203c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -558,6 +558,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3716. Node-label-expression should be included by 
     ResourceRequestPBImpl.toString. (Xianyin Xin via wangda)
 
+    YARN-3740. Fixed the typo in the configuration name:
+    APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0c491be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 1d42c60..ba07c80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1500,9 +1500,9 @@ public class YarnConfiguration extends Configuration {
    * REST API or application history protocol and shown in timeline
    * server web ui.
    */
-  public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+  public static final String APPLICATION_HISTORY_MAX_APPS =
       APPLICATION_HISTORY_PREFIX + "max-applications";
-  public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+  public static final long DEFAULT_APPLICATION_HISTORY_MAX_APPS = 10000;
 
   /** Timeline service store class */
   public static final String TIMELINE_SERVICE_STORE =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0c491be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index d3e7b5c..9bfd2d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -94,8 +94,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
     serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
         WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
     maxLoadedApplications =
-        conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
-          YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
+        conf.getLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS,
+          YarnConfiguration.DEFAULT_APPLICATION_HISTORY_MAX_APPS);
     super.serviceInit(conf);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0c491be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index ad03718..764ecb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -108,7 +108,7 @@ public class TestApplicationHistoryClientService {
     // Create a historyManager, and set the max_apps can be loaded
     // as 1.
     Configuration conf = new YarnConfiguration();
-    conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+    conf.setLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS, 1);
     ApplicationHistoryManagerOnTimelineStore historyManager2 =
         new ApplicationHistoryManagerOnTimelineStore(dataManager,
           new ApplicationACLsManager(conf));


[36/50] [abbrv] hadoop git commit: YARN-3547. FairScheduler: Apps that have no resource demand should not participate scheduling. (Xianyin Xin via kasha)

Posted by zj...@apache.org.
YARN-3547. FairScheduler: Apps that have no resource demand should not participate scheduling. (Xianyin Xin via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b126e9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b126e9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b126e9b

Branch: refs/heads/YARN-2928
Commit: 6b126e9bba94c4a8180168e2803d754b8eb81a59
Parents: 547b9dd
Author: Karthik Kambatla <ka...@apache.org>
Authored: Fri May 29 15:17:02 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:58 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../scheduler/fair/FSLeafQueue.java             | 43 ++++++++++----------
 2 files changed, 24 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b126e9b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2fe1b02..4ddf11b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -379,6 +379,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3006. Improve the error message when attempting manual failover with 
     auto-failover enabled. (Akira AJISAKA via wangda)
 
+    YARN-3547. FairScheduler: Apps that have no resource demand should not participate 
+    scheduling. (Xianyin Xin via kasha)
+
   BUG FIXES
 
     YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b126e9b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 04dbd2f..6779a1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.TreeSet;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.logging.Log;
@@ -314,35 +315,33 @@ public class FSLeafQueue extends FSQueue {
       return assigned;
     }
 
-    Comparator<Schedulable> comparator = policy.getComparator();
-    writeLock.lock();
-    try {
-      Collections.sort(runnableApps, comparator);
-    } finally {
-      writeLock.unlock();
-    }
-    // Release write lock here for better performance and avoiding deadlocks.
-    // runnableApps can be in unsorted state because of this section,
-    // but we can accept it in practice since the probability is low.
+    // Apps that have resource demands.
+    TreeSet<FSAppAttempt> pendingForResourceApps =
+        new TreeSet<FSAppAttempt>(policy.getComparator());
     readLock.lock();
     try {
-      for (FSAppAttempt sched : runnableApps) {
-        if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) {
-          continue;
-        }
-
-        assigned = sched.assignContainer(node);
-        if (!assigned.equals(Resources.none())) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Assigned container in queue:" + getName() + " " +
-                "container:" + assigned);
-          }
-          break;
+      for (FSAppAttempt app : runnableApps) {
+        Resource pending = app.getAppAttemptResourceUsage().getPending();
+        if (!pending.equals(Resources.none())) {
+          pendingForResourceApps.add(app);
         }
       }
     } finally {
       readLock.unlock();
     }
+    for (FSAppAttempt sched : pendingForResourceApps) {
+      if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) {
+        continue;
+      }
+      assigned = sched.assignContainer(node);
+      if (!assigned.equals(Resources.none())) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Assigned container in queue:" + getName() + " " +
+              "container:" + assigned);
+        }
+        break;
+      }
+    }
     return assigned;
   }
 


[20/50] [abbrv] hadoop git commit: HADOOP-7947. Validate XMLs if a relevant tool is available, when using scripts (Kengo Seki via aw)

Posted by zj...@apache.org.
HADOOP-7947. Validate XMLs if a relevant tool is available, when using scripts (Kengo Seki via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f47c0e9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f47c0e9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f47c0e9a

Branch: refs/heads/YARN-2928
Commit: f47c0e9ad030e8a15de32cec00a15e253ff52784
Parents: 82829c8
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 11:01:00 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 .../hadoop-common/src/main/bin/hadoop           |   4 +
 .../java/org/apache/hadoop/util/ConfTest.java   | 368 +++++++++++++++++++
 .../org/apache/hadoop/util/TestConfTest.java    | 204 ++++++++++
 4 files changed, 579 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f47c0e9a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8de5ae4..71b3388 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -48,6 +48,9 @@ Trunk (Unreleased)
     HADOOP-9642. Configuration to resolve environment variables via
     ${env.VARIABLE} references (Kengo Seki via aw)
 
+    HADOOP-7947. Validate XMLs if a relevant tool is available, when using
+    scripts (Kengo Seki via aw)
+
   IMPROVEMENTS
 
     HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f47c0e9a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 64c3c13..9e7b3ef 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -27,6 +27,7 @@ function hadoop_usage()
   echo "                         libraries availability"
   echo "  classpath            prints the class path needed to get the"
   echo "                         Hadoop jar and the required libraries"
+  echo "  conftest             validate configuration XML files"
   echo "  credential           interact with credential providers"
   echo "  daemonlog            get/set the log level for each daemon"
   echo "  distch path:owner:group:permisson"
@@ -127,6 +128,9 @@ case ${COMMAND} in
   classpath)
     hadoop_do_classpath_subcommand CLASS "$@"
   ;;
+  conftest)
+    CLASS=org.apache.hadoop.util.ConfTest
+  ;;
   credential)
     CLASS=org.apache.hadoop.security.alias.CredentialShell
   ;;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f47c0e9a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java
new file mode 100644
index 0000000..3f37f5a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+import java.util.Map.Entry;
+
+import javax.xml.namespace.QName;
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.Attribute;
+import javax.xml.stream.events.Characters;
+import javax.xml.stream.events.StartElement;
+import javax.xml.stream.events.XMLEvent;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * This class validates configuration XML files in ${HADOOP_CONF_DIR} or
+ * specified ones.
+ */
+@InterfaceAudience.Private
+public final class ConfTest {
+
+  private static final String USAGE =
+      "Usage: hadoop conftest [-conffile <path>|-h|--help]\n"
+      + "  Options:\n"
+      + "  \n"
+      + "  -conffile <path>\n"
+      + "    If not specified, the files in ${HADOOP_CONF_DIR}\n"
+      + "    whose name end with .xml will be verified.\n"
+      + "    If specified, that path will be verified.\n"
+      + "    You can specify either a file or directory, and\n"
+      + "    if a directory specified, the files in that directory\n"
+      + "    whose name end with .xml will be verified.\n"
+      + "    You can specify this option multiple times.\n"
+      + "  -h, --help       Print this help";
+
+  private static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR";
+
+  protected ConfTest() {
+    super();
+  }
+
+  private static List<NodeInfo> parseConf(InputStream in)
+      throws XMLStreamException {
+    QName configuration = new QName("configuration");
+    QName property = new QName("property");
+
+    List<NodeInfo> nodes = new ArrayList<NodeInfo>();
+    Stack<NodeInfo> parsed = new Stack<NodeInfo>();
+
+    XMLInputFactory factory = XMLInputFactory.newInstance();
+    XMLEventReader reader = factory.createXMLEventReader(in);
+
+    while (reader.hasNext()) {
+      XMLEvent event = reader.nextEvent();
+      if (event.isStartElement()) {
+        StartElement currentElement = event.asStartElement();
+        NodeInfo currentNode = new NodeInfo(currentElement);
+        if (parsed.isEmpty()) {
+          if (!currentElement.getName().equals(configuration)) {
+            return null;
+          }
+        } else {
+          NodeInfo parentNode = parsed.peek();
+          QName parentName = parentNode.getStartElement().getName();
+          if (parentName.equals(configuration)
+              && currentNode.getStartElement().getName().equals(property)) {
+            @SuppressWarnings("unchecked")
+            Iterator<Attribute> it = currentElement.getAttributes();
+            while (it.hasNext()) {
+              currentNode.addAttribute(it.next());
+            }
+          } else if (parentName.equals(property)) {
+            parentNode.addElement(currentElement);
+          }
+        }
+        parsed.push(currentNode);
+      } else if (event.isEndElement()) {
+        NodeInfo node = parsed.pop();
+        if (parsed.size() == 1) {
+          nodes.add(node);
+        }
+      } else if (event.isCharacters()) {
+        if (2 < parsed.size()) {
+          NodeInfo parentNode = parsed.pop();
+          StartElement parentElement = parentNode.getStartElement();
+          NodeInfo grandparentNode = parsed.peek();
+          if (grandparentNode.getElement(parentElement) == null) {
+            grandparentNode.setElement(parentElement, event.asCharacters());
+          }
+          parsed.push(parentNode);
+        }
+      }
+    }
+
+    return nodes;
+  }
+
+  public static List<String> checkConf(InputStream in) {
+    List<NodeInfo> nodes = null;
+    List<String> errors = new ArrayList<String>();
+
+    try {
+      nodes = parseConf(in);
+      if (nodes == null) {
+        errors.add("bad conf file: top-level element not <configuration>");
+      }
+    } catch (XMLStreamException e) {
+      errors.add("bad conf file: " + e.getMessage());
+    }
+
+    if (!errors.isEmpty()) {
+      return errors;
+    }
+
+    Map<String, List<Integer>> duplicatedProperties =
+        new HashMap<String, List<Integer>>();
+
+    for (NodeInfo node : nodes) {
+      StartElement element = node.getStartElement();
+      int line = element.getLocation().getLineNumber();
+
+      if (!element.getName().equals(new QName("property"))) {
+        errors.add(String.format("Line %d: element not <property>", line));
+        continue;
+      }
+
+      List<XMLEvent> events = node.getXMLEventsForQName(new QName("name"));
+      if (events == null) {
+        errors.add(String.format("Line %d: <property> has no <name>", line));
+      } else {
+        String v = null;
+        for (XMLEvent event : events) {
+          if (event.isAttribute()) {
+            v = ((Attribute) event).getValue();
+          } else {
+            Characters c = node.getElement(event.asStartElement());
+            if (c != null) {
+              v = c.getData();
+            }
+          }
+          if (v == null || v.isEmpty()) {
+            errors.add(String.format("Line %d: <property> has an empty <name>",
+                line));
+          }
+        }
+        if (v != null && !v.isEmpty()) {
+          List<Integer> lines = duplicatedProperties.get(v);
+          if (lines == null) {
+            lines = new ArrayList<Integer>();
+            duplicatedProperties.put(v, lines);
+          }
+          lines.add(node.getStartElement().getLocation().getLineNumber());
+        }
+      }
+
+      events = node.getXMLEventsForQName(new QName("value"));
+      if (events == null) {
+        errors.add(String.format("Line %d: <property> has no <value>", line));
+      }
+
+      for (QName qName : node.getDuplicatedQNames()) {
+        if (!qName.equals(new QName("source"))) {
+          errors.add(String.format("Line %d: <property> has duplicated <%s>s",
+              line, qName));
+        }
+      }
+    }
+
+    for (Entry<String, List<Integer>> e : duplicatedProperties.entrySet()) {
+      List<Integer> lines = e.getValue();
+      if (1 < lines.size()) {
+        errors.add(String.format("Line %s: duplicated <property>s for %s",
+            StringUtils.join(", ", lines), e.getKey()));
+      }
+    }
+
+    return errors;
+  }
+
+  private static File[] listFiles(File dir) {
+    return dir.listFiles(new FileFilter() {
+      @Override
+      public boolean accept(File file) {
+        return file.isFile() && file.getName().endsWith(".xml");
+      }
+    });
+  }
+
+  @SuppressWarnings("static-access")
+  public static void main(String[] args) throws IOException {
+    GenericOptionsParser genericParser = new GenericOptionsParser(args);
+    String[] remainingArgs = genericParser.getRemainingArgs();
+
+    Option conf = OptionBuilder.hasArg().create("conffile");
+    Option help = OptionBuilder.withLongOpt("help").create('h');
+    Options opts = new Options().addOption(conf).addOption(help);
+    CommandLineParser specificParser = new GnuParser();
+    CommandLine cmd = null;
+    try {
+      cmd = specificParser.parse(opts, remainingArgs);
+    } catch (MissingArgumentException e) {
+      terminate(1, "No argument specified for -conffile option");
+    } catch (ParseException e) {
+      terminate(1, USAGE);
+    }
+    if (cmd == null) {
+      terminate(1, "Failed to parse options");
+    }
+
+    if (cmd.hasOption('h')) {
+      terminate(0, USAGE);
+    }
+
+    List<File> files = new ArrayList<File>();
+    if (cmd.hasOption("conffile")) {
+      String[] values = cmd.getOptionValues("conffile");
+      for (String value : values) {
+        File confFile = new File(value);
+        if (confFile.isFile()) {
+          files.add(confFile);
+        } else if (confFile.isDirectory()) {
+          for (File file : listFiles(confFile)) {
+            files.add(file);
+          }
+        } else {
+          terminate(1, confFile.getAbsolutePath()
+              + " is neither a file nor directory");
+        }
+      }
+    } else {
+      String confDirName = System.getenv(HADOOP_CONF_DIR);
+      if (confDirName == null) {
+        terminate(1, HADOOP_CONF_DIR + " does not defined");
+      }
+      File confDir = new File(confDirName);
+      if (!confDir.isDirectory()) {
+        terminate(1, HADOOP_CONF_DIR + " is not a directory");
+      }
+      files = Arrays.asList(listFiles(confDir));
+    }
+    if (files.isEmpty()) {
+      terminate(1, "No input file to validate");
+    }
+
+    boolean ok = true;
+    for (File file : files) {
+      String path = file.getAbsolutePath();
+      List<String> errors = checkConf(new FileInputStream(file));
+      if (errors.isEmpty()) {
+        System.out.println(path + ": valid");
+      } else {
+        ok = false;
+        System.err.println(path + ":");
+        for (String error : errors) {
+          System.err.println("\t" + error);
+        }
+      }
+    }
+    if (ok) {
+      System.out.println("OK");
+    } else {
+      terminate(1, "Invalid file exists");
+    }
+  }
+
+  private static void terminate(int status, String msg) {
+    System.err.println(msg);
+    System.exit(status);
+  }
+}
+
+class NodeInfo {
+
+  private StartElement startElement;
+  private List<Attribute> attributes = new ArrayList<Attribute>();
+  private Map<StartElement, Characters> elements =
+      new HashMap<StartElement, Characters>();
+  private Map<QName, List<XMLEvent>> qNameXMLEventsMap =
+      new HashMap<QName, List<XMLEvent>>();
+
+  public NodeInfo(StartElement startElement) {
+    this.startElement = startElement;
+  }
+
+  private void addQNameXMLEvent(QName qName, XMLEvent event) {
+    List<XMLEvent> events = qNameXMLEventsMap.get(qName);
+    if (events == null) {
+      events = new ArrayList<XMLEvent>();
+      qNameXMLEventsMap.put(qName, events);
+    }
+    events.add(event);
+  }
+
+  public StartElement getStartElement() {
+    return startElement;
+  }
+
+  public void addAttribute(Attribute attribute) {
+    attributes.add(attribute);
+    addQNameXMLEvent(attribute.getName(), attribute);
+  }
+
+  public Characters getElement(StartElement element) {
+    return elements.get(element);
+  }
+
+  public void addElement(StartElement element) {
+    setElement(element, null);
+    addQNameXMLEvent(element.getName(), element);
+  }
+
+  public void setElement(StartElement element, Characters text) {
+    elements.put(element, text);
+  }
+
+  public List<QName> getDuplicatedQNames() {
+    List<QName> duplicates = new ArrayList<QName>();
+    for (Map.Entry<QName, List<XMLEvent>> e : qNameXMLEventsMap.entrySet()) {
+      if (1 < e.getValue().size()) {
+        duplicates.add(e.getKey());
+      }
+    }
+    return duplicates;
+  }
+
+  public List<XMLEvent> getXMLEventsForQName(QName qName) {
+    return qNameXMLEventsMap.get(qName);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f47c0e9a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfTest.java
new file mode 100644
index 0000000..f6cc7c3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayInputStream;
+import java.util.List;
+
+import org.junit.Test;
+
+public class TestConfTest {
+
+  @Test
+  public void testEmptyConfiguration() {
+    String conf = "<configuration/>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertTrue(errors.isEmpty());
+  }
+
+  @Test
+  public void testValidConfiguration() {
+    String conf = "<configuration>\n"
+        + "<property>\n"
+        + "<name>foo</name>\n"
+        + "<value>bar</value>\n"
+        + "</property>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertTrue(errors.isEmpty());
+  }
+
+  @Test
+  public void testSourceDuplicationIsValid() {
+    String conf = "<configuration>\n"
+        + "<property source='a'>\n"
+        + "<name>foo</name>\n"
+        + "<value>bar</value>\n"
+        + "<source>b</source>\n"
+        + "<source>c</source>\n"
+        + "</property>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertTrue(errors.isEmpty());
+  }
+
+  @Test
+  public void testEmptyInput() {
+    String conf = "";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertTrue(errors.get(0).startsWith("bad conf file: "));
+  }
+
+  @Test
+  public void testInvalidFormat() {
+    String conf = "<configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertTrue(errors.get(0).startsWith("bad conf file: "));
+  }
+
+  @Test
+  public void testRootElementNotConfiguration() {
+    String conf = "<configurations/>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertEquals("bad conf file: top-level element not <configuration>", errors.get(0));
+  }
+
+  @Test
+  public void testSubElementNotProperty() {
+    String conf = "<configuration>\n"
+        + "<foo/>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertEquals("Line 2: element not <property>", errors.get(0));
+  }
+
+  @Test
+  public void testPropertyHasNoName() {
+    String conf ="<configuration>\n"
+        + "<property>\n"
+        + "<value>foo</value>\n"
+        + "</property>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertEquals("Line 2: <property> has no <name>", errors.get(0));
+  }
+
+  @Test
+  public void testPropertyHasEmptyName() {
+    String conf = "<configuration>\n"
+        + "<property>\n"
+        + "<name></name>\n"
+        + "<value>foo</value>\n"
+        + "</property>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertEquals("Line 2: <property> has an empty <name>", errors.get(0));
+  }
+
+  @Test
+  public void testPropertyHasNoValue() {
+    String conf ="<configuration>\n"
+        + "<property>\n"
+        + "<name>foo</name>\n"
+        + "</property>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertEquals("Line 2: <property> has no <value>", errors.get(0));
+  }
+
+  @Test
+  public void testPropertyHasEmptyValue() {
+    String conf = "<configuration>\n"
+        + "<property>\n"
+        + "<name>foo</name>\n"
+        + "<value></value>\n"
+        + "</property>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertTrue(errors.isEmpty());
+  }
+
+  @Test
+  public void testPropertyHasDuplicatedAttributeAndElement() {
+    String conf = "<configuration>\n"
+        + "<property name='foo'>\n"
+        + "<name>bar</name>\n"
+        + "<value>baz</value>\n"
+        + "</property>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertEquals("Line 2: <property> has duplicated <name>s", errors.get(0));
+  }
+
+  @Test
+  public void testPropertyHasDuplicatedElements() {
+    String conf = "<configuration>\n"
+        + "<property>\n"
+        + "<name>foo</name>\n"
+        + "<name>bar</name>\n"
+        + "<value>baz</value>\n"
+        + "</property>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertEquals("Line 2: <property> has duplicated <name>s", errors.get(0));
+  }
+
+  @Test
+  public void testDuplicatedProperties() {
+    String conf = "<configuration>\n"
+        + "<property>\n"
+        + "<name>foo</name>\n"
+        + "<value>bar</value>\n"
+        + "</property>\n"
+        + "<property>\n"
+        + "<name>foo</name>\n"
+        + "<value>baz</value>\n"
+        + "</property>\n"
+        + "</configuration>";
+    ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+    List<String> errors = ConfTest.checkConf(bais);
+    assertEquals(1, errors.size());
+    assertEquals("Line 2, 6: duplicated <property>s for foo", errors.get(0));
+  }
+
+}


[47/50] [abbrv] hadoop git commit: HADOOP-12037. Fix wrong classname in example configuration of hadoop-auth documentation. Contributed by Masatake Iwasaki.

Posted by zj...@apache.org.
HADOOP-12037. Fix wrong classname in example configuration of hadoop-auth documentation. Contributed by Masatake Iwasaki.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77c1692d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77c1692d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77c1692d

Branch: refs/heads/YARN-2928
Commit: 77c1692d010ad893bf40805192ac0aa9aa6049d2
Parents: 065ab4d
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Jun 1 18:04:52 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:22 2015 -0700

----------------------------------------------------------------------
 .../src/site/markdown/Configuration.md            | 18 +++++++++++++++---
 hadoop-common-project/hadoop-common/CHANGES.txt   |  3 +++
 2 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/77c1692d/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md b/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
index 9d076bb..2f9b860 100644
--- a/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
+++ b/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
@@ -73,12 +73,13 @@ To use Kerberos SPNEGO as the authentication mechanism, the authentication filte
 
 **Example**:
 
+```xml
     <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
         ...
 
         <filter>
             <filter-name>kerberosFilter</filter-name>
-            <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
+            <filter-class>org.apache.hadoop.security.authentication.server.AuthenticationFilter</filter-class>
             <init-param>
                 <param-name>type</param-name>
                 <param-value>kerberos</param-value>
@@ -112,6 +113,7 @@ To use Kerberos SPNEGO as the authentication mechanism, the authentication filte
 
         ...
     </web-app>
+```
 
 ### Pseudo/Simple Configuration
 
@@ -125,12 +127,13 @@ To use Pseudo/Simple as the authentication mechanism (trusting the value of the
 
 **Example**:
 
+```xml
     <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
         ...
 
         <filter>
             <filter-name>simpleFilter</filter-name>
-            <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
+            <filter-class>org.apache.hadoop.security.authentication.server.AuthenticationFilter</filter-class>
             <init-param>
                 <param-name>type</param-name>
                 <param-value>simple</param-value>
@@ -160,6 +163,7 @@ To use Pseudo/Simple as the authentication mechanism (trusting the value of the
 
         ...
     </web-app>
+```
 
 ### AltKerberos Configuration
 
@@ -175,12 +179,13 @@ The AltKerberos authentication mechanism is a partially implemented derivative o
 
 **Example**:
 
+```xml
     <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
         ...
 
         <filter>
             <filter-name>kerberosFilter</filter-name>
-            <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
+            <filter-class>org.apache.hadoop.security.authentication.server.AuthenticationFilter</filter-class>
             <init-param>
                 <param-name>type</param-name>
                 <param-value>org.my.subclass.of.AltKerberosAuthenticationHandler</param-value>
@@ -218,6 +223,7 @@ The AltKerberos authentication mechanism is a partially implemented derivative o
 
         ...
     </web-app>
+```
 
 ### SignerSecretProvider Configuration
 
@@ -262,6 +268,7 @@ The following configuration properties are specific to the `zookeeper` implement
 
 **Example**:
 
+```xml
     <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
         ...
 
@@ -279,9 +286,11 @@ The following configuration properties are specific to the `zookeeper` implement
 
         ...
     </web-app>
+```
 
 **Example**:
 
+```xml
     <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
         ...
 
@@ -299,9 +308,11 @@ The following configuration properties are specific to the `zookeeper` implement
 
         ...
     </web-app>
+```
 
 **Example**:
 
+```xml
     <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
         ...
 
@@ -339,3 +350,4 @@ The following configuration properties are specific to the `zookeeper` implement
 
         ...
     </web-app>
+```

http://git-wip-us.apache.org/repos/asf/hadoop/blob/77c1692d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 44c2969..a0e6e90 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -631,6 +631,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12043. Display warning if defaultFs is not set when running fs
     commands. (Lei Xu via wang)
 
+    HADOOP-12037. Fix wrong classname in example configuration of hadoop-auth
+    documentation. (Masatake Iwasaki via wang)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp


[07/50] [abbrv] hadoop git commit: YARN-3722. Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils. Contributed by Masatake Iwasaki.

Posted by zj...@apache.org.
YARN-3722. Merge multiple TestWebAppUtils into
o.a.h.yarn.webapp.util.TestWebAppUtils. Contributed by Masatake Iwasaki.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/822cac2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/822cac2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/822cac2a

Branch: refs/heads/YARN-2928
Commit: 822cac2a90637b221d7c4357f8630167223bd982
Parents: 2b39cff
Author: Devaraj K <de...@apache.org>
Authored: Thu May 28 15:35:06 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:53 2015 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 +
 .../hadoop/yarn/util/TestWebAppUtils.java       | 81 --------------------
 .../yarn/webapp/util/TestWebAppUtils.java       | 52 +++++++++++++
 3 files changed, 55 insertions(+), 81 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/822cac2a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5ce8c63..5326f2d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -365,6 +365,9 @@ Release 2.8.0 - UNRELEASED
     YARN-3700. Made generic history service load a number of latest applications
     according to the parameter or the configuration. (Xuan Gong via zjshen)
 
+    YARN-3722. Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils.
+    (Masatake Iwasaki via devaraj)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

http://git-wip-us.apache.org/repos/asf/hadoop/blob/822cac2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
deleted file mode 100644
index 169787f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class TestWebAppUtils {
-
-  private static final String RM1_NODE_ID = "rm1";
-  private static final String RM2_NODE_ID = "rm2";
-
-  // Because WebAppUtils#getResolvedAddress tries to resolve the hostname, we add a static mapping for dummy hostnames
-  // to make this test run anywhere without having to give some resolvable hostnames
-  private static String dummyHostNames[] = {"host1", "host2", "host3"};
-  private static final String anyIpAddress = "1.2.3.4";
-  private static Map<String, String> savedStaticResolution = new HashMap<>();
-
-  @BeforeClass
-  public static void initializeDummyHostnameResolution() throws Exception {
-    String previousIpAddress;
-    for (String hostName : dummyHostNames) {
-      if (null != (previousIpAddress = NetUtils.getStaticResolution(hostName))) {
-        savedStaticResolution.put(hostName, previousIpAddress);
-      }
-      NetUtils.addStaticResolution(hostName, anyIpAddress);
-    }
-  }
-
-  @AfterClass
-  public static void restoreDummyHostnameResolution() throws Exception {
-    for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution.entrySet()) {
-      NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), hostnameToIpEntry.getValue());
-    }
-  }
-
-  @Test
-  public void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
-    Configuration configuration = new Configuration();
-    final String rmAddress = "host1:8088";
-    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, rmAddress);
-    final String rm1Address = "host2:8088";
-    final String rm2Address = "host3:8088";
-    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID, rm1Address);
-    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address);
-    configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
-    configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
-
-    String rmRemoteUrl = WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(configuration);
-    Assert.assertEquals("ResolvedRemoteRMWebAppUrl should resolve to the first HA RM address", rm1Address, rmRemoteUrl);
-
-    String rmLocalUrl = WebAppUtils.getResolvedRMWebAppURLWithoutScheme(configuration);
-    Assert.assertEquals("ResolvedRMWebAppUrl should resolve to the default RM webapp address", rmAddress, rmLocalUrl);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/822cac2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
index 2bd91b4..dcc8ba4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
@@ -23,17 +23,69 @@ import static org.junit.Assert.assertEquals;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.http.HttpServer2;
 import org.apache.hadoop.http.HttpServer2.Builder;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Assert;
 import org.junit.Test;
 
 public class TestWebAppUtils {
+  private static final String RM1_NODE_ID = "rm1";
+  private static final String RM2_NODE_ID = "rm2";
+
+  // Because WebAppUtils#getResolvedAddress tries to resolve the hostname, we add a static mapping for dummy hostnames
+  // to make this test run anywhere without having to give some resolvable hostnames
+  private static String dummyHostNames[] = {"host1", "host2", "host3"};
+  private static final String anyIpAddress = "1.2.3.4";
+  private static Map<String, String> savedStaticResolution = new HashMap<>();
+
+  @BeforeClass
+  public static void initializeDummyHostnameResolution() throws Exception {
+    String previousIpAddress;
+    for (String hostName : dummyHostNames) {
+      if (null != (previousIpAddress = NetUtils.getStaticResolution(hostName))) {
+        savedStaticResolution.put(hostName, previousIpAddress);
+      }
+      NetUtils.addStaticResolution(hostName, anyIpAddress);
+    }
+  }
+
+  @AfterClass
+  public static void restoreDummyHostnameResolution() throws Exception {
+    for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution.entrySet()) {
+      NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), hostnameToIpEntry.getValue());
+    }
+  }
+
+  @Test
+  public void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
+    Configuration configuration = new Configuration();
+    final String rmAddress = "host1:8088";
+    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, rmAddress);
+    final String rm1Address = "host2:8088";
+    final String rm2Address = "host3:8088";
+    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID, rm1Address);
+    configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address);
+    configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+    configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
+
+    String rmRemoteUrl = WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(configuration);
+    Assert.assertEquals("ResolvedRemoteRMWebAppUrl should resolve to the first HA RM address", rm1Address, rmRemoteUrl);
+
+    String rmLocalUrl = WebAppUtils.getResolvedRMWebAppURLWithoutScheme(configuration);
+    Assert.assertEquals("ResolvedRMWebAppUrl should resolve to the default RM webapp address", rmAddress, rmLocalUrl);
+  }
 
   @Test
   public void testGetPassword() throws Exception {