You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/06/03 01:47:33 UTC
[01/50] [abbrv] hadoop git commit: MAPREDUCE-6336. Enable v2
FileOutputCommitter by default. (Siqi Li via gera)
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 a9738ceb1 -> 2e1248088
MAPREDUCE-6336. Enable v2 FileOutputCommitter by default. (Siqi Li via gera)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/315bde8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/315bde8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/315bde8e
Branch: refs/heads/YARN-2928
Commit: 315bde8e25f755cfabbc2069da87f3570b6016ae
Parents: 41d3f67
Author: Gera Shegalov <ge...@apache.org>
Authored: Wed May 27 13:11:10 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 15:46:58 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/main/resources/mapred-default.xml | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/315bde8e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 221d98c..c0df835 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -37,6 +37,9 @@ Trunk (Unreleased)
MAPREDUCE-4424. 'mapred job -list' command should show the job name
as well. (Avinash Kujur via aajisaka)
+ MAPREDUCE-6336. Enable v2 FileOutputCommitter by default.
+ (Siqi Li via gera)
+
NEW FEATURES
MAPREDUCE-778. Rumen Anonymizer. (Amar Kamat and Chris Douglas via amarrk)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/315bde8e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index acb45a7..bc41443 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1330,7 +1330,7 @@
<property>
<name>mapreduce.fileoutputcommitter.algorithm.version</name>
- <value>1</value>
+ <value>2</value>
<description>The file output committer algorithm version
valid algorithm version number: 1 or 2
default to 1, which is the original algorithm
[22/50] [abbrv] hadoop git commit: Fixed more FilesSystemRMStateStore
issues. Contributed by Vinod Kumar Vavilapalli.
Posted by zj...@apache.org.
Fixed more FilesSystemRMStateStore issues. Contributed by Vinod Kumar Vavilapalli.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14b30520
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14b30520
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14b30520
Branch: refs/heads/YARN-2928
Commit: 14b30520f9ca9d0f5555f594430f70d45e6d60b6
Parents: 3fcb33b
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu May 28 15:25:12 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 12 +++++++++++
.../recovery/FileSystemRMStateStore.java | 20 +++++++++----------
.../recovery/TestFSRMStateStore.java | 21 ++++++++++----------
3 files changed, 32 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b30520/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6892e2d..1d42c60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -269,6 +269,18 @@ public class YarnConfiguration extends Configuration {
/** ACL used in case none is found. Allows nothing. */
public static final String DEFAULT_YARN_APP_ACL = " ";
+ /**
+ * Enable/disable intermediate-data encryption at YARN level. For now, this
+ * only is used by the FileSystemRMStateStore to setup right file-system
+ * security attributes.
+ */
+ @Private
+ public static final String YARN_INTERMEDIATE_DATA_ENCRYPTION = YARN_PREFIX
+ + "intermediate-data-encryption.enable";
+
+ @Private
+ public static final Boolean DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION = false;
+
/** The address of the RM admin interface.*/
public static final String RM_ADMIN_ADDRESS =
RM_PREFIX + "admin.address";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b30520/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 0d97d6f..b972791 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AM
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
+
import com.google.common.annotations.VisibleForTesting;
@Private
@@ -100,7 +101,8 @@ public class FileSystemRMStateStore extends RMStateStore {
private Path dtSequenceNumberPath = null;
private int fsNumRetries;
private long fsRetryInterval;
- private volatile boolean isHDFS;
+ private boolean intermediateEncryptionEnabled =
+ YarnConfiguration.DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION;
@VisibleForTesting
Path fsWorkingPath;
@@ -121,6 +123,9 @@ public class FileSystemRMStateStore extends RMStateStore {
fsRetryInterval =
conf.getLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_INTERVAL_MS);
+ intermediateEncryptionEnabled =
+ conf.getBoolean(YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION,
+ YarnConfiguration.DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION);
}
@Override
@@ -145,17 +150,11 @@ public class FileSystemRMStateStore extends RMStateStore {
}
fs = fsWorkingPath.getFileSystem(fsConf);
- isHDFS = fs.getScheme().toLowerCase().contains("hdfs");
mkdirsWithRetries(rmDTSecretManagerRoot);
mkdirsWithRetries(rmAppRoot);
mkdirsWithRetries(amrmTokenSecretManagerRoot);
}
- @VisibleForTesting
- void setIsHDFS(boolean isHDFS) {
- this.isHDFS = isHDFS;
- }
-
@Override
protected synchronized void closeInternal() throws Exception {
closeWithRetries();
@@ -856,10 +855,11 @@ public class FileSystemRMStateStore extends RMStateStore {
private void setUnreadableBySuperuserXattrib(Path p)
throws IOException {
- if (isHDFS &&
- !fs.getXAttrs(p).containsKey(UNREADABLE_BY_SUPERUSER_XATTRIB)) {
+ if (fs.getScheme().toLowerCase().contains("hdfs")
+ && intermediateEncryptionEnabled
+ && !fs.getXAttrs(p).containsKey(UNREADABLE_BY_SUPERUSER_XATTRIB)) {
fs.setXAttr(p, UNREADABLE_BY_SUPERUSER_XATTRIB, null,
- EnumSet.of(XAttrSetFlag.CREATE));
+ EnumSet.of(XAttrSetFlag.CREATE));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b30520/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
index 2dba8b4..c842294 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
@@ -26,11 +26,6 @@ import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -38,17 +33,22 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Assert;
import org.junit.Test;
public class TestFSRMStateStore extends RMStateStoreTestBase {
@@ -111,6 +111,10 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8);
conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS,
900L);
+ if (adminCheckEnable) {
+ conf.setBoolean(
+ YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true);
+ }
this.store = new TestFileSystemRMStore(conf);
Assert.assertEquals(store.getNumRetries(), 8);
Assert.assertEquals(store.getRetryInterval(), 900L);
@@ -119,11 +123,6 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
store.startInternal();
Assert.assertTrue(store.fs != previousFs);
Assert.assertTrue(store.fs.getConf() == store.fsConf);
- if (adminCheckEnable) {
- store.setIsHDFS(true);
- } else {
- store.setIsHDFS(false);
- }
return store;
}
[29/50] [abbrv] hadoop git commit: YARN-3716. Node-label-expression
should be included by ResourceRequestPBImpl.toString. (Xianyin Xin via
wangda)
Posted by zj...@apache.org.
YARN-3716. Node-label-expression should be included by ResourceRequestPBImpl.toString. (Xianyin Xin via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/219d73d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/219d73d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/219d73d8
Branch: refs/heads/YARN-2928
Commit: 219d73d826685145bf759cb51178f432a1b626d5
Parents: 14b3052
Author: Wangda Tan <wa...@apache.org>
Authored: Thu May 28 16:34:18 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/219d73d8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1dd4ee3..2fe1b02 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -549,6 +549,9 @@ Release 2.8.0 - UNRELEASED
YARN-3632. Ordering policy should be allowed to reorder an application when
demand changes. (Craig Welch via jianhe)
+ YARN-3716. Node-label-expression should be included by
+ ResourceRequestPBImpl.toString. (Xianyin Xin via wangda)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/219d73d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index 791e6a3..fc09cef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -184,7 +184,8 @@ public class ResourceRequestPBImpl extends ResourceRequest {
return "{Priority: " + getPriority() + ", Capability: " + getCapability()
+ ", # Containers: " + getNumContainers()
+ ", Location: " + getResourceName()
- + ", Relax Locality: " + getRelaxLocality() + "}";
+ + ", Relax Locality: " + getRelaxLocality()
+ + ", Node Label Expression: " + getNodeLabelExpression() + "}";
}
@Override
[32/50] [abbrv] hadoop git commit: HDFS-8443. Document
dfs.namenode.service.handler.count in hdfs-site.xml. Contributed by
J.Andreina.
Posted by zj...@apache.org.
HDFS-8443. Document dfs.namenode.service.handler.count in hdfs-site.xml. Contributed by J.Andreina.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdc6ac8a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdc6ac8a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdc6ac8a
Branch: refs/heads/YARN-2928
Commit: fdc6ac8a1da468f098a32f8e15e812b6ec67a4d6
Parents: 219d73d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri May 29 11:36:08 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 16 +++++++++++++++-
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdc6ac8a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1c9b25e..514076b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -585,6 +585,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8482. Rename BlockInfoContiguous to BlockInfo. (Zhe Zhang via wang)
+ HDFS-8443. Document dfs.namenode.service.handler.count in hdfs-site.xml.
+ (J.Andreina via aajisaka)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdc6ac8a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 7f0730b..7b579cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -602,7 +602,21 @@
<property>
<name>dfs.namenode.handler.count</name>
<value>10</value>
- <description>The number of server threads for the namenode.</description>
+ <description>The number of Namenode RPC server threads that listen to
+ requests from clients.
+ If dfs.namenode.servicerpc-address is not configured then
+ Namenode RPC server threads listen to requests from all nodes.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.service.handler.count</name>
+ <value>10</value>
+ <description>The number of Namenode RPC server threads that listen to
+ requests from DataNodes and from all other non-client nodes.
+ dfs.namenode.service.handler.count will be valid only if
+ dfs.namenode.servicerpc-address is configured.
+ </description>
</property>
<property>
[04/50] [abbrv] hadoop git commit: HDFS-8482. Rename
BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 476ff36..9acd81f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
@@ -942,7 +942,7 @@ public class FSEditLogLoader {
*/
private void addNewBlock(FSDirectory fsDir, AddBlockOp op, INodeFile file)
throws IOException {
- BlockInfoContiguous[] oldBlocks = file.getBlocks();
+ BlockInfo[] oldBlocks = file.getBlocks();
Block pBlock = op.getPenultimateBlock();
Block newBlock= op.getLastBlock();
@@ -968,7 +968,7 @@ public class FSEditLogLoader {
Preconditions.checkState(oldBlocks == null || oldBlocks.length == 0);
}
// add the new block
- BlockInfoContiguous newBI = new BlockInfoContiguousUnderConstruction(
+ BlockInfo newBI = new BlockInfoContiguousUnderConstruction(
newBlock, file.getPreferredBlockReplication());
fsNamesys.getBlockManager().addBlockCollection(newBI, file);
file.addBlock(newBI);
@@ -982,7 +982,7 @@ public class FSEditLogLoader {
private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
INodesInPath iip, INodeFile file) throws IOException {
// Update its block list
- BlockInfoContiguous[] oldBlocks = file.getBlocks();
+ BlockInfo[] oldBlocks = file.getBlocks();
Block[] newBlocks = op.getBlocks();
String path = op.getPath();
@@ -991,7 +991,7 @@ public class FSEditLogLoader {
// First, update blocks in common
for (int i = 0; i < oldBlocks.length && i < newBlocks.length; i++) {
- BlockInfoContiguous oldBlock = oldBlocks[i];
+ BlockInfo oldBlock = oldBlocks[i];
Block newBlock = newBlocks[i];
boolean isLastBlock = i == newBlocks.length - 1;
@@ -1043,7 +1043,7 @@ public class FSEditLogLoader {
// We're adding blocks
for (int i = oldBlocks.length; i < newBlocks.length; i++) {
Block newBlock = newBlocks[i];
- BlockInfoContiguous newBI;
+ BlockInfo newBI;
if (!op.shouldCompleteLastBlock()) {
// TODO: shouldn't this only be true for the last block?
// what about an old-version fsync() where fsync isn't called
@@ -1055,7 +1055,7 @@ public class FSEditLogLoader {
// is only executed when loading edits written by prior
// versions of Hadoop. Current versions always log
// OP_ADD operations as each block is allocated.
- newBI = new BlockInfoContiguous(newBlock,
+ newBI = new BlockInfo(newBlock,
file.getPreferredBlockReplication());
}
fsNamesys.getBlockManager().addBlockCollection(newBI, file);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index ec2babd..a46f2b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -52,7 +52,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutFlags;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -686,7 +686,7 @@ public class FSImageFormat {
public void updateBlocksMap(INodeFile file) {
// Add file->block mapping
- final BlockInfoContiguous[] blocks = file.getBlocks();
+ final BlockInfo[] blocks = file.getBlocks();
if (blocks != null) {
final BlockManager bm = namesystem.getBlockManager();
for (int i = 0; i < blocks.length; i++) {
@@ -753,9 +753,9 @@ public class FSImageFormat {
// file
// read blocks
- BlockInfoContiguous[] blocks = new BlockInfoContiguous[numBlocks];
+ BlockInfo[] blocks = new BlockInfo[numBlocks];
for (int j = 0; j < numBlocks; j++) {
- blocks[j] = new BlockInfoContiguous(replication);
+ blocks[j] = new BlockInfo(replication);
blocks[j].readFields(in);
}
@@ -775,7 +775,7 @@ public class FSImageFormat {
clientMachine = FSImageSerialization.readString(in);
// convert the last block to BlockUC
if (blocks.length > 0) {
- BlockInfoContiguous lastBlk = blocks[blocks.length - 1];
+ BlockInfo lastBlk = blocks[blocks.length - 1];
blocks[blocks.length - 1] = new BlockInfoContiguousUnderConstruction(
lastBlk, replication);
}
@@ -958,9 +958,9 @@ public class FSImageFormat {
FileUnderConstructionFeature uc = cons.getFileUnderConstructionFeature();
oldnode.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
if (oldnode.numBlocks() > 0) {
- BlockInfoContiguous ucBlock = cons.getLastBlock();
+ BlockInfo ucBlock = cons.getLastBlock();
// we do not replace the inode, just replace the last block of oldnode
- BlockInfoContiguous info = namesystem.getBlockManager().addBlockCollection(
+ BlockInfo info = namesystem.getBlockManager().addBlockCollection(
ucBlock, oldnode);
oldnode.setBlock(oldnode.numBlocks() - 1, info);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index d966c69..29713a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -43,7 +42,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
@@ -211,7 +210,7 @@ public final class FSImageFormatPBINode {
public static void updateBlocksMap(INodeFile file, BlockManager bm) {
// Add file->block mapping
- final BlockInfoContiguous[] blocks = file.getBlocks();
+ final BlockInfo[] blocks = file.getBlocks();
if (blocks != null) {
for (int i = 0; i < blocks.length; i++) {
file.setBlock(i, bm.addBlockCollection(blocks[i], file));
@@ -324,9 +323,9 @@ public final class FSImageFormatPBINode {
short replication = (short) f.getReplication();
LoaderContext state = parent.getLoaderContext();
- BlockInfoContiguous[] blocks = new BlockInfoContiguous[bp.size()];
+ BlockInfo[] blocks = new BlockInfo[bp.size()];
for (int i = 0, e = bp.size(); i < e; ++i) {
- blocks[i] = new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
+ blocks[i] = new BlockInfo(PBHelper.convert(bp.get(i)), replication);
}
final PermissionStatus permissions = loadPermission(f.getPermission(),
parent.getLoaderContext().getStringTable());
@@ -352,7 +351,7 @@ public final class FSImageFormatPBINode {
INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
if (blocks.length > 0) {
- BlockInfoContiguous lastBlk = file.getLastBlock();
+ BlockInfo lastBlk = file.getLastBlock();
// replace the last block of file
file.setBlock(file.numBlocks() - 1, new BlockInfoContiguousUnderConstruction(
lastBlk, replication));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 1888d87..d790b8f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
@@ -126,12 +126,12 @@ public class FSImageSerialization {
long preferredBlockSize = in.readLong();
int numBlocks = in.readInt();
- BlockInfoContiguous[] blocks = new BlockInfoContiguous[numBlocks];
+ BlockInfo[] blocks = new BlockInfo[numBlocks];
Block blk = new Block();
int i = 0;
for (; i < numBlocks-1; i++) {
blk.readFields(in);
- blocks[i] = new BlockInfoContiguous(blk, blockReplication);
+ blocks[i] = new BlockInfo(blk, blockReplication);
}
// last block is UNDER_CONSTRUCTION
if(numBlocks > 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index bfd6eba..798f8d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -203,7 +203,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretMan
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager.SecretManagerState;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
@@ -2016,7 +2016,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
// Check if the file is already being truncated with the same length
- final BlockInfoContiguous last = file.getLastBlock();
+ final BlockInfo last = file.getLastBlock();
if (last != null && last.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
final Block truncateBlock
= ((BlockInfoContiguousUnderConstruction)last).getTruncateBlock();
@@ -2090,7 +2090,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
leaseManager.addLease(
file.getFileUnderConstructionFeature().getClientName(), file.getId());
boolean shouldRecoverNow = (newBlock == null);
- BlockInfoContiguous oldBlock = file.getLastBlock();
+ BlockInfo oldBlock = file.getLastBlock();
boolean shouldCopyOnTruncate = shouldCopyOnTruncate(file, oldBlock);
if(newBlock == null) {
newBlock = (shouldCopyOnTruncate) ? createNewBlock() :
@@ -2141,7 +2141,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* Defines if a replica needs to be copied on truncate or
* can be truncated in place.
*/
- boolean shouldCopyOnTruncate(INodeFile file, BlockInfoContiguous blk) {
+ boolean shouldCopyOnTruncate(INodeFile file, BlockInfo blk) {
if(!isUpgradeFinalized()) {
return true;
}
@@ -2505,7 +2505,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
recoverLeaseInternal(RecoverLeaseOp.APPEND_FILE, iip, src, holder,
clientMachine, false);
- final BlockInfoContiguous lastBlock = myFile.getLastBlock();
+ final BlockInfo lastBlock = myFile.getLastBlock();
// Check that the block has at least minimum replication.
if(lastBlock != null && lastBlock.isComplete() &&
!getBlockManager().isSufficientlyReplicated(lastBlock)) {
@@ -2561,7 +2561,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
} else {
- BlockInfoContiguous lastBlock = file.getLastBlock();
+ BlockInfo lastBlock = file.getLastBlock();
if (lastBlock != null) {
ExtendedBlock blk = new ExtendedBlock(this.getBlockPoolId(), lastBlock);
ret = new LocatedBlock(blk, new DatanodeInfo[0]);
@@ -2603,7 +2603,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/** Compute quota change for converting a complete block to a UC block */
private QuotaCounts computeQuotaDeltaForUCBlock(INodeFile file) {
final QuotaCounts delta = new QuotaCounts.Builder().build();
- final BlockInfoContiguous lastBlock = file.getLastBlock();
+ final BlockInfo lastBlock = file.getLastBlock();
if (lastBlock != null) {
final long diff = file.getPreferredBlockSize() - lastBlock.getNumBytes();
final short repl = file.getPreferredBlockReplication();
@@ -2740,7 +2740,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
op.getExceptionMessage(src, holder, clientMachine,
"lease recovery is in progress. Try again later."));
} else {
- final BlockInfoContiguous lastBlock = file.getLastBlock();
+ final BlockInfo lastBlock = file.getLastBlock();
if (lastBlock != null
&& lastBlock.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
throw new RecoveryInProgressException(
@@ -3065,10 +3065,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
.getBlocks());
} else {
// check the penultimate block of this file
- BlockInfoContiguous b = v.getPenultimateBlock();
+ BlockInfo b = v.getPenultimateBlock();
return b == null ||
blockManager.checkBlocksProperlyReplicated(
- src, new BlockInfoContiguous[] { b });
+ src, new BlockInfo[] { b });
}
}
@@ -3242,7 +3242,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
for (Block b : blocks.getToDeleteList()) {
if (trackBlockCounts) {
- BlockInfoContiguous bi = getStoredBlock(b);
+ BlockInfo bi = getStoredBlock(b);
if (bi.isComplete()) {
numRemovedComplete++;
if (bi.numNodes() >= blockManager.minReplication) {
@@ -3466,10 +3466,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final INodeFile pendingFile = iip.getLastINode().asFile();
int nrBlocks = pendingFile.numBlocks();
- BlockInfoContiguous[] blocks = pendingFile.getBlocks();
+ BlockInfo[] blocks = pendingFile.getBlocks();
int nrCompleteBlocks;
- BlockInfoContiguous curBlock = null;
+ BlockInfo curBlock = null;
for(nrCompleteBlocks = 0; nrCompleteBlocks < nrBlocks; nrCompleteBlocks++) {
curBlock = blocks[nrCompleteBlocks];
if(!curBlock.isComplete())
@@ -3504,9 +3504,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// The last block is not COMPLETE, and
// that the penultimate block if exists is either COMPLETE or COMMITTED
- final BlockInfoContiguous lastBlock = pendingFile.getLastBlock();
+ final BlockInfo lastBlock = pendingFile.getLastBlock();
BlockUCState lastBlockState = lastBlock.getBlockUCState();
- BlockInfoContiguous penultimateBlock = pendingFile.getPenultimateBlock();
+ BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
// If penultimate block doesn't exist then its minReplication is met
boolean penultimateBlockMinReplication = penultimateBlock == null ? true :
@@ -3650,7 +3650,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
@VisibleForTesting
- BlockInfoContiguous getStoredBlock(Block block) {
+ BlockInfo getStoredBlock(Block block) {
return blockManager.getStoredBlock(block);
}
@@ -3710,7 +3710,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkNameNodeSafeMode(
"Cannot commitBlockSynchronization while in safe mode");
- final BlockInfoContiguous storedBlock = getStoredBlock(
+ final BlockInfo storedBlock = getStoredBlock(
ExtendedBlock.getLocalBlock(oldBlock));
if (storedBlock == null) {
if (deleteblock) {
@@ -3871,7 +3871,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* @throws IOException on error
*/
@VisibleForTesting
- String closeFileCommitBlocks(INodeFile pendingFile, BlockInfoContiguous storedBlock)
+ String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock)
throws IOException {
final INodesInPath iip = INodesInPath.fromINode(pendingFile);
final String src = iip.getPath();
@@ -4162,7 +4162,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
while (it.hasNext()) {
Block b = it.next();
- BlockInfoContiguous blockInfo = blockManager.getStoredBlock(b);
+ BlockInfo blockInfo = blockManager.getStoredBlock(b);
if (blockInfo.getBlockCollection().getStoragePolicyID()
== lpPolicy.getId()) {
filesToDelete.add(blockInfo.getBlockCollection());
@@ -5105,7 +5105,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
SafeModeInfo safeMode = this.safeMode;
if (safeMode == null) // mostly true
return;
- BlockInfoContiguous storedBlock = getStoredBlock(b);
+ BlockInfo storedBlock = getStoredBlock(b);
if (storedBlock.isComplete()) {
safeMode.decrementSafeBlockCount((short)blockManager.countNodes(b).liveReplicas());
}
@@ -5665,7 +5665,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
+ "access token for block " + block);
// check stored block state
- BlockInfoContiguous storedBlock = getStoredBlock(ExtendedBlock.getLocalBlock(block));
+ BlockInfo storedBlock = getStoredBlock(ExtendedBlock.getLocalBlock(block));
if (storedBlock == null ||
storedBlock.getBlockUCState() != BlockUCState.UNDER_CONSTRUCTION) {
throw new IOException(block +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
index 1ebdde6..d07ae1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@@ -58,7 +58,7 @@ public class FileUnderConstructionFeature implements INode.Feature {
*/
void updateLengthOfLastBlock(INodeFile f, long lastBlockLength)
throws IOException {
- BlockInfoContiguous lastBlock = f.getLastBlock();
+ BlockInfo lastBlock = f.getLastBlock();
assert (lastBlock != null) : "The last block for path "
+ f.getFullPathName() + " is null when updating its length";
assert (lastBlock instanceof BlockInfoContiguousUnderConstruction)
@@ -74,7 +74,7 @@ public class FileUnderConstructionFeature implements INode.Feature {
*/
void cleanZeroSizeBlock(final INodeFile f,
final BlocksMapUpdateInfo collectedBlocks) {
- final BlockInfoContiguous[] blocks = f.getBlocks();
+ final BlockInfo[] blocks = f.getBlocks();
if (blocks != null && blocks.length > 0
&& blocks[blocks.length - 1] instanceof BlockInfoContiguousUnderConstruction) {
BlockInfoContiguousUnderConstruction lastUC =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index a6ff6fb..294323c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
@@ -121,17 +121,17 @@ public class INodeFile extends INodeWithAdditionalFields
private long header = 0L;
- private BlockInfoContiguous[] blocks;
+ private BlockInfo[] blocks;
INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
- long atime, BlockInfoContiguous[] blklist, short replication,
+ long atime, BlockInfo[] blklist, short replication,
long preferredBlockSize) {
this(id, name, permissions, mtime, atime, blklist, replication,
preferredBlockSize, (byte) 0);
}
INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
- long atime, BlockInfoContiguous[] blklist, short replication,
+ long atime, BlockInfo[] blklist, short replication,
long preferredBlockSize, byte storagePolicyID) {
super(id, name, permissions, mtime, atime);
header = HeaderFormat.toLong(preferredBlockSize, replication, storagePolicyID);
@@ -226,13 +226,13 @@ public class INodeFile extends INodeWithAdditionalFields
}
@Override // BlockCollection
- public void setBlock(int index, BlockInfoContiguous blk) {
+ public void setBlock(int index, BlockInfo blk) {
this.blocks[index] = blk;
}
@Override // BlockCollection, the file should be under construction
public BlockInfoContiguousUnderConstruction setLastBlock(
- BlockInfoContiguous lastBlock, DatanodeStorageInfo[] locations)
+ BlockInfo lastBlock, DatanodeStorageInfo[] locations)
throws IOException {
Preconditions.checkState(isUnderConstruction(),
"file is no longer under construction");
@@ -265,7 +265,7 @@ public class INodeFile extends INodeWithAdditionalFields
BlockInfoContiguousUnderConstruction uc =
(BlockInfoContiguousUnderConstruction)blocks[size_1];
//copy to a new list
- BlockInfoContiguous[] newlist = new BlockInfoContiguous[size_1];
+ BlockInfo[] newlist = new BlockInfo[size_1];
System.arraycopy(blocks, 0, newlist, 0, size_1);
setBlocks(newlist);
return uc;
@@ -420,16 +420,16 @@ public class INodeFile extends INodeWithAdditionalFields
/** @return the blocks of the file. */
@Override
- public BlockInfoContiguous[] getBlocks() {
+ public BlockInfo[] getBlocks() {
return this.blocks;
}
/** @return blocks of the file corresponding to the snapshot. */
- public BlockInfoContiguous[] getBlocks(int snapshot) {
+ public BlockInfo[] getBlocks(int snapshot) {
if(snapshot == CURRENT_STATE_ID || getDiffs() == null)
return getBlocks();
FileDiff diff = getDiffs().getDiffById(snapshot);
- BlockInfoContiguous[] snapshotBlocks =
+ BlockInfo[] snapshotBlocks =
diff == null ? getBlocks() : diff.getBlocks();
if(snapshotBlocks != null)
return snapshotBlocks;
@@ -441,7 +441,7 @@ public class INodeFile extends INodeWithAdditionalFields
void updateBlockCollection() {
if (blocks != null) {
- for(BlockInfoContiguous b : blocks) {
+ for(BlockInfo b : blocks) {
b.setBlockCollection(this);
}
}
@@ -457,8 +457,8 @@ public class INodeFile extends INodeWithAdditionalFields
totalAddedBlocks += f.blocks.length;
}
- BlockInfoContiguous[] newlist =
- new BlockInfoContiguous[size + totalAddedBlocks];
+ BlockInfo[] newlist =
+ new BlockInfo[size + totalAddedBlocks];
System.arraycopy(this.blocks, 0, newlist, 0, size);
for(INodeFile in: inodes) {
@@ -473,12 +473,12 @@ public class INodeFile extends INodeWithAdditionalFields
/**
* add a block to the block list
*/
- void addBlock(BlockInfoContiguous newblock) {
+ void addBlock(BlockInfo newblock) {
if (this.blocks == null) {
- this.setBlocks(new BlockInfoContiguous[]{newblock});
+ this.setBlocks(new BlockInfo[]{newblock});
} else {
int size = this.blocks.length;
- BlockInfoContiguous[] newlist = new BlockInfoContiguous[size + 1];
+ BlockInfo[] newlist = new BlockInfo[size + 1];
System.arraycopy(this.blocks, 0, newlist, 0, size);
newlist[size] = newblock;
this.setBlocks(newlist);
@@ -486,7 +486,7 @@ public class INodeFile extends INodeWithAdditionalFields
}
/** Set the blocks. */
- public void setBlocks(BlockInfoContiguous[] blocks) {
+ public void setBlocks(BlockInfo[] blocks) {
this.blocks = blocks;
}
@@ -538,7 +538,7 @@ public class INodeFile extends INodeWithAdditionalFields
public void clearFile(ReclaimContext reclaimContext) {
if (blocks != null && reclaimContext.collectedBlocks != null) {
- for (BlockInfoContiguous blk : blocks) {
+ for (BlockInfo blk : blocks) {
reclaimContext.collectedBlocks.addDeleteBlock(blk);
blk.setBlockCollection(null);
}
@@ -710,16 +710,16 @@ public class INodeFile extends INodeWithAdditionalFields
*/
public final QuotaCounts storagespaceConsumed(BlockStoragePolicy bsp) {
QuotaCounts counts = new QuotaCounts.Builder().build();
- final Iterable<BlockInfoContiguous> blocks;
+ final Iterable<BlockInfo> blocks;
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if (sf == null) {
blocks = Arrays.asList(getBlocks());
} else {
// Collect all distinct blocks
- Set<BlockInfoContiguous> allBlocks = new HashSet<>(Arrays.asList(getBlocks()));
+ Set<BlockInfo> allBlocks = new HashSet<>(Arrays.asList(getBlocks()));
List<FileDiff> diffs = sf.getDiffs().asList();
for(FileDiff diff : diffs) {
- BlockInfoContiguous[] diffBlocks = diff.getBlocks();
+ BlockInfo[] diffBlocks = diff.getBlocks();
if (diffBlocks != null) {
allBlocks.addAll(Arrays.asList(diffBlocks));
}
@@ -728,7 +728,7 @@ public class INodeFile extends INodeWithAdditionalFields
}
final short replication = getPreferredBlockReplication();
- for (BlockInfoContiguous b : blocks) {
+ for (BlockInfo b : blocks) {
long blockSize = b.isComplete() ? b.getNumBytes() :
getPreferredBlockSize();
counts.addStorageSpace(blockSize * replication);
@@ -747,7 +747,7 @@ public class INodeFile extends INodeWithAdditionalFields
/**
* Return the penultimate allocated block for this file.
*/
- BlockInfoContiguous getPenultimateBlock() {
+ BlockInfo getPenultimateBlock() {
if (blocks == null || blocks.length <= 1) {
return null;
}
@@ -755,7 +755,7 @@ public class INodeFile extends INodeWithAdditionalFields
}
@Override
- public BlockInfoContiguous getLastBlock() {
+ public BlockInfo getLastBlock() {
return blocks == null || blocks.length == 0? null: blocks[blocks.length-1];
}
@@ -782,7 +782,7 @@ public class INodeFile extends INodeWithAdditionalFields
*/
public long collectBlocksBeyondMax(final long max,
final BlocksMapUpdateInfo collectedBlocks) {
- final BlockInfoContiguous[] oldBlocks = getBlocks();
+ final BlockInfo[] oldBlocks = getBlocks();
if (oldBlocks == null)
return 0;
// find the minimum n such that the size of the first n blocks > max
@@ -814,17 +814,17 @@ public class INodeFile extends INodeWithAdditionalFields
void computeQuotaDeltaForTruncate(
long newLength, BlockStoragePolicy bsps,
QuotaCounts delta) {
- final BlockInfoContiguous[] blocks = getBlocks();
+ final BlockInfo[] blocks = getBlocks();
if (blocks == null || blocks.length == 0) {
return;
}
long size = 0;
- for (BlockInfoContiguous b : blocks) {
+ for (BlockInfo b : blocks) {
size += b.getNumBytes();
}
- BlockInfoContiguous[] sblocks = null;
+ BlockInfo[] sblocks = null;
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if (sf != null) {
FileDiff diff = sf.getDiffs().getLast();
@@ -833,7 +833,7 @@ public class INodeFile extends INodeWithAdditionalFields
for (int i = blocks.length - 1; i >= 0 && size > newLength;
size -= blocks[i].getNumBytes(), --i) {
- BlockInfoContiguous bi = blocks[i];
+ BlockInfo bi = blocks[i];
long truncatedBytes;
if (size - newLength < bi.getNumBytes()) {
// Record a full block as the last block will be copied during
@@ -863,20 +863,20 @@ public class INodeFile extends INodeWithAdditionalFields
}
void truncateBlocksTo(int n) {
- final BlockInfoContiguous[] newBlocks;
+ final BlockInfo[] newBlocks;
if (n == 0) {
- newBlocks = BlockInfoContiguous.EMPTY_ARRAY;
+ newBlocks = BlockInfo.EMPTY_ARRAY;
} else {
- newBlocks = new BlockInfoContiguous[n];
+ newBlocks = new BlockInfo[n];
System.arraycopy(getBlocks(), 0, newBlocks, 0, n);
}
// set new blocks
setBlocks(newBlocks);
}
- public void collectBlocksBeyondSnapshot(BlockInfoContiguous[] snapshotBlocks,
+ public void collectBlocksBeyondSnapshot(BlockInfo[] snapshotBlocks,
BlocksMapUpdateInfo collectedBlocks) {
- BlockInfoContiguous[] oldBlocks = getBlocks();
+ BlockInfo[] oldBlocks = getBlocks();
if(snapshotBlocks == null || oldBlocks == null)
return;
// Skip blocks in common between the file and the snapshot
@@ -900,7 +900,7 @@ public class INodeFile extends INodeWithAdditionalFields
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if(sf == null)
return;
- BlockInfoContiguous[] snapshotBlocks =
+ BlockInfo[] snapshotBlocks =
getDiffs().findEarlierSnapshotBlocks(snapshotId);
if(snapshotBlocks == null)
return;
@@ -914,12 +914,12 @@ public class INodeFile extends INodeWithAdditionalFields
/**
* @return true if the block is contained in a snapshot or false otherwise.
*/
- boolean isBlockInLatestSnapshot(BlockInfoContiguous block) {
+ boolean isBlockInLatestSnapshot(BlockInfo block) {
FileWithSnapshotFeature sf = this.getFileWithSnapshotFeature();
if (sf == null || sf.getDiffs() == null) {
return false;
}
- BlockInfoContiguous[] snapshotBlocks = getDiffs()
+ BlockInfo[] snapshotBlocks = getDiffs()
.findEarlierSnapshotBlocks(getDiffs().getLastSnapshotId());
return snapshotBlocks != null &&
Arrays.asList(snapshotBlocks).contains(block);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 0806f82..f954a58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -33,7 +33,7 @@ import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.util.Daemon;
@@ -108,11 +108,11 @@ public class LeaseManager {
for (Long id : getINodeIdWithLeases()) {
final INodeFile cons = fsnamesystem.getFSDirectory().getInode(id).asFile();
Preconditions.checkState(cons.isUnderConstruction());
- BlockInfoContiguous[] blocks = cons.getBlocks();
+ BlockInfo[] blocks = cons.getBlocks();
if(blocks == null) {
continue;
}
- for(BlockInfoContiguous b : blocks) {
+ for(BlockInfo b : blocks) {
if(!b.isComplete())
numUCBlocks++;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 0daf367..39180b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -64,7 +64,7 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactor
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
@@ -243,7 +243,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
//get blockInfo
Block block = new Block(Block.getBlockId(blockId));
//find which file this block belongs to
- BlockInfoContiguous blockInfo = bm.getStoredBlock(block);
+ BlockInfo blockInfo = bm.getStoredBlock(block);
if(blockInfo == null) {
out.println("Block "+ blockId +" " + NONEXISTENT_STATUS);
LOG.warn("Block "+ blockId + " " + NONEXISTENT_STATUS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index c4cbbc1..363c208 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.AclEntryStatusFormat;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -239,13 +239,13 @@ public class FSImageFormatPBSnapshot {
FileDiff diff = new FileDiff(pbf.getSnapshotId(), copy, null,
pbf.getFileSize());
List<BlockProto> bpl = pbf.getBlocksList();
- BlockInfoContiguous[] blocks = new BlockInfoContiguous[bpl.size()];
+ BlockInfo[] blocks = new BlockInfo[bpl.size()];
for(int j = 0, e = bpl.size(); j < e; ++j) {
Block blk = PBHelper.convert(bpl.get(j));
- BlockInfoContiguous storedBlock = fsn.getBlockManager().getStoredBlock(blk);
+ BlockInfo storedBlock = fsn.getBlockManager().getStoredBlock(blk);
if(storedBlock == null) {
storedBlock = fsn.getBlockManager().addBlockCollection(
- new BlockInfoContiguous(blk, copy.getFileReplication()), file);
+ new BlockInfo(blk, copy.getFileReplication()), file);
}
blocks[j] = storedBlock;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
index 48c5f33..52bf58e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
@@ -21,7 +21,7 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@@ -38,7 +38,7 @@ public class FileDiff extends
/** The file size at snapshot creation time. */
private final long fileSize;
/** A copy of the INodeFile block list. Used in truncate. */
- private BlockInfoContiguous[] blocks;
+ private BlockInfo[] blocks;
FileDiff(int snapshotId, INodeFile file) {
super(snapshotId, null, null);
@@ -64,7 +64,7 @@ public class FileDiff extends
* up to the current {@link #fileSize}.
* Should be done only once.
*/
- public void setBlocks(BlockInfoContiguous[] blocks) {
+ public void setBlocks(BlockInfo[] blocks) {
if(this.blocks != null)
return;
int numBlocks = 0;
@@ -73,7 +73,7 @@ public class FileDiff extends
this.blocks = Arrays.copyOf(blocks, numBlocks);
}
- public BlockInfoContiguous[] getBlocks() {
+ public BlockInfo[] getBlocks() {
return blocks;
}
@@ -118,7 +118,7 @@ public class FileDiff extends
if (blocks == null || collectedBlocks == null) {
return;
}
- for (BlockInfoContiguous blk : blocks) {
+ for (BlockInfo blk : blocks) {
collectedBlocks.addDeleteBlock(blk);
}
blocks = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
index 62aaccd..6b8388e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
@@ -21,7 +21,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.INode;
@@ -59,14 +59,14 @@ public class FileDiffList extends
}
}
- public BlockInfoContiguous[] findEarlierSnapshotBlocks(int snapshotId) {
+ public BlockInfo[] findEarlierSnapshotBlocks(int snapshotId) {
assert snapshotId != Snapshot.NO_SNAPSHOT_ID : "Wrong snapshot id";
if (snapshotId == Snapshot.CURRENT_STATE_ID) {
return null;
}
List<FileDiff> diffs = this.asList();
int i = Collections.binarySearch(diffs, snapshotId);
- BlockInfoContiguous[] blocks = null;
+ BlockInfo[] blocks = null;
for(i = i >= 0 ? i : -i-2; i >= 0; i--) {
blocks = diffs.get(i).getBlocks();
if(blocks != null) {
@@ -76,14 +76,14 @@ public class FileDiffList extends
return blocks;
}
- public BlockInfoContiguous[] findLaterSnapshotBlocks(int snapshotId) {
+ public BlockInfo[] findLaterSnapshotBlocks(int snapshotId) {
assert snapshotId != Snapshot.NO_SNAPSHOT_ID : "Wrong snapshot id";
if (snapshotId == Snapshot.CURRENT_STATE_ID) {
return null;
}
List<FileDiff> diffs = this.asList();
int i = Collections.binarySearch(diffs, snapshotId);
- BlockInfoContiguous[] blocks = null;
+ BlockInfo[] blocks = null;
for (i = i >= 0 ? i+1 : -i-1; i < diffs.size(); i++) {
blocks = diffs.get(i).getBlocks();
if (blocks != null) {
@@ -100,7 +100,7 @@ public class FileDiffList extends
*/
void combineAndCollectSnapshotBlocks(
INode.ReclaimContext reclaimContext, INodeFile file, FileDiff removed) {
- BlockInfoContiguous[] removedBlocks = removed.getBlocks();
+ BlockInfo[] removedBlocks = removed.getBlocks();
if (removedBlocks == null) {
FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature();
assert sf != null : "FileWithSnapshotFeature is null";
@@ -114,10 +114,10 @@ public class FileDiffList extends
if (earlierDiff != null) {
earlierDiff.setBlocks(removedBlocks);
}
- BlockInfoContiguous[] earlierBlocks =
- (earlierDiff == null ? new BlockInfoContiguous[]{} : earlierDiff.getBlocks());
+ BlockInfo[] earlierBlocks =
+ (earlierDiff == null ? new BlockInfo[]{} : earlierDiff.getBlocks());
// Find later snapshot (or file itself) with blocks
- BlockInfoContiguous[] laterBlocks = findLaterSnapshotBlocks(removed.getSnapshotId());
+ BlockInfo[] laterBlocks = findLaterSnapshotBlocks(removed.getSnapshotId());
laterBlocks = (laterBlocks==null) ? file.getBlocks() : laterBlocks;
// Skip blocks, which belong to either the earlier or the later lists
int i = 0;
@@ -129,7 +129,7 @@ public class FileDiffList extends
break;
}
// Check if last block is part of truncate recovery
- BlockInfoContiguous lastBlock = file.getLastBlock();
+ BlockInfo lastBlock = file.getLastBlock();
Block dontRemoveBlock = null;
if (lastBlock != null && lastBlock.getBlockUCState().equals(
HdfsServerConstants.BlockUCState.UNDER_RECOVERY)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
index 555a662..f017d4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.AclStorage;
@@ -208,7 +208,7 @@ public class FileWithSnapshotFeature implements INode.Feature {
// Collect blocks that should be deleted
FileDiff last = diffs.getLast();
- BlockInfoContiguous[] snapshotBlocks = last == null ? null : last.getBlocks();
+ BlockInfo[] snapshotBlocks = last == null ? null : last.getBlocks();
if(snapshotBlocks == null)
file.collectBlocksBeyondMax(max, reclaimContext.collectedBlocks());
else
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index a88a459..50b85c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -108,7 +108,7 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
@@ -1609,7 +1609,7 @@ public class DFSTestUtil {
public static DatanodeDescriptor getExpectedPrimaryNode(NameNode nn,
ExtendedBlock blk) {
BlockManager bm0 = nn.getNamesystem().getBlockManager();
- BlockInfoContiguous storedBlock = bm0.getStoredBlock(blk.getLocalBlock());
+ BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock());
assertTrue("Block " + blk + " should be under construction, " +
"got: " + storedBlock,
storedBlock instanceof BlockInfoContiguousUnderConstruction);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index 1ab7427..1f18014 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
@@ -671,7 +671,7 @@ public class TestDecommission {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
- BlockInfoContiguous info =
+ BlockInfo info =
blockManager.getStoredBlock(b.getLocalBlock());
int count = 0;
StringBuilder sb = new StringBuilder("Replica locations: ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index c5662ca..96d4d8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -49,7 +49,7 @@ public class TestBlockInfo {
@Test
public void testIsDeleted() {
- BlockInfoContiguous blockInfo = new BlockInfoContiguous((short) 3);
+ BlockInfo blockInfo = new BlockInfo((short) 3);
BlockCollection bc = Mockito.mock(BlockCollection.class);
blockInfo.setBlockCollection(bc);
Assert.assertFalse(blockInfo.isDeleted());
@@ -59,7 +59,7 @@ public class TestBlockInfo {
@Test
public void testAddStorage() throws Exception {
- BlockInfoContiguous blockInfo = new BlockInfoContiguous((short) 3);
+ BlockInfo blockInfo = new BlockInfo((short) 3);
final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1");
@@ -71,9 +71,9 @@ public class TestBlockInfo {
@Test
public void testCopyConstructor() {
- BlockInfoContiguous old = new BlockInfoContiguous((short) 3);
+ BlockInfo old = new BlockInfo((short) 3);
try {
- BlockInfoContiguous copy = new BlockInfoContiguous(old);
+ BlockInfo copy = new BlockInfo(old);
assertEquals(old.getBlockCollection(), copy.getBlockCollection());
assertEquals(old.getCapacity(), copy.getCapacity());
} catch (Exception e) {
@@ -88,11 +88,11 @@ public class TestBlockInfo {
final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1", "127.0.0.1");
final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2"));
final int NUM_BLOCKS = 10;
- BlockInfoContiguous[] blockInfos = new BlockInfoContiguous[NUM_BLOCKS];
+ BlockInfo[] blockInfos = new BlockInfo[NUM_BLOCKS];
// Create a few dummy blocks and add them to the first storage.
for (int i = 0; i < NUM_BLOCKS; ++i) {
- blockInfos[i] = new BlockInfoContiguous((short) 3);
+ blockInfos[i] = new BlockInfo((short) 3);
storage1.addBlock(blockInfos[i]);
}
@@ -111,14 +111,14 @@ public class TestBlockInfo {
DatanodeStorageInfo dd = DFSTestUtil.createDatanodeStorageInfo("s1", "1.1.1.1");
ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
- ArrayList<BlockInfoContiguous> blockInfoList = new ArrayList<BlockInfoContiguous>();
+ ArrayList<BlockInfo> blockInfoList = new ArrayList<BlockInfo>();
int headIndex;
int curIndex;
LOG.info("Building block list...");
for (int i = 0; i < MAX_BLOCKS; i++) {
blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
- blockInfoList.add(new BlockInfoContiguous(blockList.get(i), (short) 3));
+ blockInfoList.add(new BlockInfo(blockList.get(i), (short) 3));
dd.addBlock(blockInfoList.get(i));
// index of the datanode should be 0
@@ -129,7 +129,7 @@ public class TestBlockInfo {
// list length should be equal to the number of blocks we inserted
LOG.info("Checking list length...");
assertEquals("Length should be MAX_BLOCK", MAX_BLOCKS, dd.numBlocks());
- Iterator<BlockInfoContiguous> it = dd.getBlockIterator();
+ Iterator<BlockInfo> it = dd.getBlockIterator();
int len = 0;
while (it.hasNext()) {
it.next();
@@ -151,7 +151,7 @@ public class TestBlockInfo {
// move head of the list to the head - this should not change the list
LOG.info("Moving head to the head...");
- BlockInfoContiguous temp = dd.getBlockListHeadForTesting();
+ BlockInfo temp = dd.getBlockListHeadForTesting();
curIndex = 0;
headIndex = 0;
dd.moveBlockToHead(temp, curIndex, headIndex);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 58210c1..bf46ed7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -147,7 +147,7 @@ public class TestBlockManager {
private void doBasicTest(int testIndex) {
List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
List<DatanodeDescriptor> origNodes = getNodes(origStorages);
- BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+ BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
DatanodeStorageInfo[] pipeline = scheduleSingleReplication(blockInfo);
assertEquals(2, pipeline.length);
@@ -179,7 +179,7 @@ public class TestBlockManager {
// Block originally on A1, A2, B1
List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
List<DatanodeDescriptor> origNodes = getNodes(origStorages);
- BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+ BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
// Decommission two of the nodes (A1, A2)
List<DatanodeDescriptor> decomNodes = startDecommission(0, 1);
@@ -223,7 +223,7 @@ public class TestBlockManager {
// Block originally on A1, A2, B1
List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
List<DatanodeDescriptor> origNodes = getNodes(origStorages);
- BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+ BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
// Decommission all of the nodes
List<DatanodeDescriptor> decomNodes = startDecommission(0, 1, 3);
@@ -276,7 +276,7 @@ public class TestBlockManager {
// Block originally on A1, A2, B1
List<DatanodeStorageInfo> origStorages = getStorages(0, 1, 3);
List<DatanodeDescriptor> origNodes = getNodes(origStorages);
- BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+ BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
// Decommission all of the nodes in rack A
List<DatanodeDescriptor> decomNodes = startDecommission(0, 1, 2);
@@ -335,7 +335,7 @@ public class TestBlockManager {
private void doTestSufficientlyReplBlocksUsesNewRack(int testIndex) {
// Originally on only nodes in rack A.
List<DatanodeDescriptor> origNodes = rackA;
- BlockInfoContiguous blockInfo = addBlockOnNodes(testIndex, origNodes);
+ BlockInfo blockInfo = addBlockOnNodes(testIndex, origNodes);
DatanodeStorageInfo pipeline[] = scheduleSingleReplication(blockInfo);
assertEquals(2, pipeline.length); // single new copy
@@ -378,7 +378,7 @@ public class TestBlockManager {
* Tell the block manager that replication is completed for the given
* pipeline.
*/
- private void fulfillPipeline(BlockInfoContiguous blockInfo,
+ private void fulfillPipeline(BlockInfo blockInfo,
DatanodeStorageInfo[] pipeline) throws IOException {
for (int i = 1; i < pipeline.length; i++) {
DatanodeStorageInfo storage = pipeline[i];
@@ -387,9 +387,9 @@ public class TestBlockManager {
}
}
- private BlockInfoContiguous blockOnNodes(long blkId, List<DatanodeDescriptor> nodes) {
+ private BlockInfo blockOnNodes(long blkId, List<DatanodeDescriptor> nodes) {
Block block = new Block(blkId);
- BlockInfoContiguous blockInfo = new BlockInfoContiguous(block, (short) 3);
+ BlockInfo blockInfo = new BlockInfo(block, (short) 3);
for (DatanodeDescriptor dn : nodes) {
for (DatanodeStorageInfo storage : dn.getStorageInfos()) {
@@ -431,10 +431,10 @@ public class TestBlockManager {
return nodes;
}
- private BlockInfoContiguous addBlockOnNodes(long blockId, List<DatanodeDescriptor> nodes) {
+ private BlockInfo addBlockOnNodes(long blockId, List<DatanodeDescriptor> nodes) {
BlockCollection bc = Mockito.mock(BlockCollection.class);
Mockito.doReturn((short)3).when(bc).getPreferredBlockReplication();
- BlockInfoContiguous blockInfo = blockOnNodes(blockId, nodes);
+ BlockInfo blockInfo = blockOnNodes(blockId, nodes);
bm.blocksMap.addBlockCollection(blockInfo, bc);
return blockInfo;
@@ -677,21 +677,21 @@ public class TestBlockManager {
// blk_42 is finalized.
long receivedBlockId = 42; // arbitrary
- BlockInfoContiguous receivedBlock = addBlockToBM(receivedBlockId);
+ BlockInfo receivedBlock = addBlockToBM(receivedBlockId);
rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivedBlock),
ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
builder.add(new FinalizedReplica(receivedBlock, null, null));
// blk_43 is under construction.
long receivingBlockId = 43;
- BlockInfoContiguous receivingBlock = addUcBlockToBM(receivingBlockId);
+ BlockInfo receivingBlock = addUcBlockToBM(receivingBlockId);
rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingBlock),
ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
builder.add(new ReplicaBeingWritten(receivingBlock, null, null, null));
// blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record.
long receivingReceivedBlockId = 44;
- BlockInfoContiguous receivingReceivedBlock = addBlockToBM(receivingReceivedBlockId);
+ BlockInfo receivingReceivedBlock = addBlockToBM(receivingReceivedBlockId);
rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
@@ -709,7 +709,7 @@ public class TestBlockManager {
// blk_46 exists in DN for a long time, so it's in full BR, but not in IBR.
long existedBlockId = 46;
- BlockInfoContiguous existedBlock = addBlockToBM(existedBlockId);
+ BlockInfo existedBlock = addBlockToBM(existedBlockId);
builder.add(new FinalizedReplica(existedBlock, null, null));
// process IBR and full BR
@@ -735,17 +735,17 @@ public class TestBlockManager {
(ds) >= 0);
}
- private BlockInfoContiguous addBlockToBM(long blkId) {
+ private BlockInfo addBlockToBM(long blkId) {
Block block = new Block(blkId);
- BlockInfoContiguous blockInfo =
- new BlockInfoContiguous(block, (short) 3);
+ BlockInfo blockInfo =
+ new BlockInfo(block, (short) 3);
BlockCollection bc = Mockito.mock(BlockCollection.class);
Mockito.doReturn((short) 3).when(bc).getPreferredBlockReplication();
bm.blocksMap.addBlockCollection(blockInfo, bc);
return blockInfo;
}
- private BlockInfoContiguous addUcBlockToBM(long blkId) {
+ private BlockInfo addUcBlockToBM(long blkId) {
Block block = new Block(blkId);
BlockInfoContiguousUnderConstruction blockInfo =
new BlockInfoContiguousUnderConstruction(block, (short) 3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
index 7cdb423..fe639e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
@@ -58,8 +58,8 @@ public class TestDatanodeDescriptor {
public void testBlocksCounter() throws Exception {
DatanodeDescriptor dd = BlockManagerTestUtil.getLocalDatanodeDescriptor(true);
assertEquals(0, dd.numBlocks());
- BlockInfoContiguous blk = new BlockInfoContiguous(new Block(1L), (short) 1);
- BlockInfoContiguous blk1 = new BlockInfoContiguous(new Block(2L), (short) 2);
+ BlockInfo blk = new BlockInfo(new Block(1L), (short) 1);
+ BlockInfo blk1 = new BlockInfo(new Block(2L), (short) 2);
DatanodeStorageInfo[] storages = dd.getStorageInfos();
assertTrue(storages.length > 0);
// add first block
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
index 259404e..844c5d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
@@ -159,7 +159,7 @@ public class TestPendingReplication {
DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, TIMEOUT);
MiniDFSCluster cluster = null;
Block block;
- BlockInfoContiguous blockInfo;
+ BlockInfo blockInfo;
try {
cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_COUNT).build();
@@ -178,7 +178,7 @@ public class TestPendingReplication {
//
block = new Block(1, 1, 0);
- blockInfo = new BlockInfoContiguous(block, (short) 3);
+ blockInfo = new BlockInfo(block, (short) 3);
pendingReplications.increment(block,
DatanodeStorageInfo.toDatanodeDescriptors(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 3226578..f317723 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -34,7 +34,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
@@ -1219,7 +1218,7 @@ public class TestReplicationPolicy {
chosenBlocks = underReplicatedBlocks.chooseUnderReplicatedBlocks(1);
assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
- final BlockInfoContiguous info = new BlockInfoContiguous(block1, (short) 1);
+ final BlockInfo info = new BlockInfo(block1, (short) 1);
final BlockCollection mbc = mock(BlockCollection.class);
when(mbc.getLastBlock()).thenReturn(info);
when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1);
@@ -1241,12 +1240,12 @@ public class TestReplicationPolicy {
when(dn.isDecommissioned()).thenReturn(true);
when(storage.getState()).thenReturn(DatanodeStorage.State.NORMAL);
when(storage.getDatanodeDescriptor()).thenReturn(dn);
- when(storage.removeBlock(any(BlockInfoContiguous.class))).thenReturn(true);
- when(storage.addBlock(any(BlockInfoContiguous.class))).thenReturn
+ when(storage.removeBlock(any(BlockInfo.class))).thenReturn(true);
+ when(storage.addBlock(any(BlockInfo.class))).thenReturn
(DatanodeStorageInfo.AddBlockResult.ADDED);
ucBlock.addStorage(storage);
- when(mbc.setLastBlock((BlockInfoContiguous) any(), (DatanodeStorageInfo[]) any()))
+ when(mbc.setLastBlock((BlockInfo) any(), (DatanodeStorageInfo[]) any()))
.thenReturn(ucBlock);
bm.convertLastBlockToUnderConstruction(mbc, 0L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index 0349251..3f96c0c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.Storage;
@@ -66,10 +66,10 @@ public class CreateEditsLog {
INodeDirectory dirInode = new INodeDirectory(inodeId.nextValue(), null, p,
0L);
editLog.logMkDir(BASE_PATH, dirInode);
- BlockInfoContiguous[] blocks = new BlockInfoContiguous[blocksPerFile];
+ BlockInfo[] blocks = new BlockInfo[blocksPerFile];
for (int iB = 0; iB < blocksPerFile; ++iB) {
blocks[iB] =
- new BlockInfoContiguous(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
+ new BlockInfo(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
replication);
}
@@ -97,7 +97,7 @@ public class CreateEditsLog {
editLog.logMkDir(currentDir, dirInode);
}
INodeFile fileUc = new INodeFile(inodeId.nextValue(), null,
- p, 0L, 0L, BlockInfoContiguous.EMPTY_ARRAY, replication, blockSize);
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
fileUc.toUnderConstruction("", "");
editLog.logOpenFile(filePath, fileUc, false, false);
editLog.logCloseFile(filePath, inode);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
index a417c3d..301ee25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlock.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.junit.After;
import org.junit.Before;
@@ -87,21 +87,21 @@ public class TestAddBlock {
// check file1
INodeFile file1Node = fsdir.getINode4Write(file1.toString()).asFile();
- BlockInfoContiguous[] file1Blocks = file1Node.getBlocks();
+ BlockInfo[] file1Blocks = file1Node.getBlocks();
assertEquals(1, file1Blocks.length);
assertEquals(BLOCKSIZE - 1, file1Blocks[0].getNumBytes());
assertEquals(BlockUCState.COMPLETE, file1Blocks[0].getBlockUCState());
// check file2
INodeFile file2Node = fsdir.getINode4Write(file2.toString()).asFile();
- BlockInfoContiguous[] file2Blocks = file2Node.getBlocks();
+ BlockInfo[] file2Blocks = file2Node.getBlocks();
assertEquals(1, file2Blocks.length);
assertEquals(BLOCKSIZE, file2Blocks[0].getNumBytes());
assertEquals(BlockUCState.COMPLETE, file2Blocks[0].getBlockUCState());
// check file3
INodeFile file3Node = fsdir.getINode4Write(file3.toString()).asFile();
- BlockInfoContiguous[] file3Blocks = file3Node.getBlocks();
+ BlockInfo[] file3Blocks = file3Node.getBlocks();
assertEquals(2, file3Blocks.length);
assertEquals(BLOCKSIZE, file3Blocks[0].getNumBytes());
assertEquals(BlockUCState.COMPLETE, file3Blocks[0].getBlockUCState());
@@ -110,7 +110,7 @@ public class TestAddBlock {
// check file4
INodeFile file4Node = fsdir.getINode4Write(file4.toString()).asFile();
- BlockInfoContiguous[] file4Blocks = file4Node.getBlocks();
+ BlockInfo[] file4Blocks = file4Node.getBlocks();
assertEquals(2, file4Blocks.length);
assertEquals(BLOCKSIZE, file4Blocks[0].getNumBytes());
assertEquals(BlockUCState.COMPLETE, file4Blocks[0].getBlockUCState());
@@ -141,7 +141,7 @@ public class TestAddBlock {
FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
INodeFile fileNode = fsdir.getINode4Write(file1.toString()).asFile();
- BlockInfoContiguous[] fileBlocks = fileNode.getBlocks();
+ BlockInfo[] fileBlocks = fileNode.getBlocks();
assertEquals(2, fileBlocks.length);
assertEquals(BLOCKSIZE, fileBlocks[0].getNumBytes());
assertEquals(BlockUCState.COMPLETE, fileBlocks[0].getBlockUCState());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
index 1fbe160..f372bec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockUnderConstruction.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.hdfs.TestFileCreation;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
@@ -91,12 +91,12 @@ public class TestBlockUnderConstruction {
" isUnderConstruction = " + inode.isUnderConstruction() +
" expected to be " + isFileOpen,
inode.isUnderConstruction() == isFileOpen);
- BlockInfoContiguous[] blocks = inode.getBlocks();
+ BlockInfo[] blocks = inode.getBlocks();
assertTrue("File does not have blocks: " + inode.toString(),
blocks != null && blocks.length > 0);
int idx = 0;
- BlockInfoContiguous curBlock;
+ BlockInfo curBlock;
// all blocks but the last two should be regular blocks
for(; idx < blocks.length - 2; idx++) {
curBlock = blocks[idx];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index ea560fe..8c9da01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -78,7 +78,7 @@ public class TestCommitBlockSynchronization {
doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class));
doReturn(blockInfo).when(file).getLastBlock();
doReturn("").when(namesystemSpy).closeFileCommitBlocks(
- any(INodeFile.class), any(BlockInfoContiguous.class));
+ any(INodeFile.class), any(BlockInfo.class));
doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog();
return namesystemSpy;
@@ -106,7 +106,7 @@ public class TestCommitBlockSynchronization {
lastBlock, genStamp, length, false, false, newTargets, null);
// Simulate 'completing' the block.
- BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
+ BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
completedBlockInfo.setBlockCollection(file);
completedBlockInfo.setGenerationStamp(genStamp);
doReturn(completedBlockInfo).when(namesystemSpy)
@@ -178,7 +178,7 @@ public class TestCommitBlockSynchronization {
namesystemSpy.commitBlockSynchronization(
lastBlock, genStamp, length, true, false, newTargets, null);
- BlockInfoContiguous completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
+ BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
completedBlockInfo.setBlockCollection(file);
completedBlockInfo.setGenerationStamp(genStamp);
doReturn(completedBlockInfo).when(namesystemSpy)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 1e42e34..f654107 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -69,7 +69,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -205,7 +205,7 @@ public class TestEditLog {
for (int i = 0; i < numTransactions; i++) {
INodeFile inode = new INodeFile(namesystem.dir.allocateNewInodeId(), null,
- p, 0L, 0L, BlockInfoContiguous.EMPTY_ARRAY, replication, blockSize);
+ p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize);
inode.toUnderConstruction("", "");
editLog.logOpenFile("/filename" + (startIndex + i), inode, false, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 5653df5..27a1bd3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
@@ -105,7 +105,7 @@ public class TestFSImage {
INodeFile file2Node = fsn.dir.getINode4Write(file2.toString()).asFile();
assertEquals("hello".length(), file2Node.computeFileSize());
assertTrue(file2Node.isUnderConstruction());
- BlockInfoContiguous[] blks = file2Node.getBlocks();
+ BlockInfo[] blks = file2Node.getBlocks();
assertEquals(1, blks.length);
assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
// check lease manager
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 7f31f84..eabd0c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -79,7 +79,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
@@ -885,7 +885,7 @@ public class TestFsck {
// intentionally corrupt NN data structure
INodeFile node = (INodeFile) cluster.getNamesystem().dir.getINode
(fileName, true);
- final BlockInfoContiguous[] blocks = node.getBlocks();
+ final BlockInfo[] blocks = node.getBlocks();
assertEquals(blocks.length, 1);
blocks[0].setNumBytes(-1L); // set the block length to be negative
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
index 4d0f994..7a3a8d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
@@ -21,7 +21,7 @@ import org.apache.commons.io.Charsets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -125,7 +125,7 @@ public class TestGetBlockLocations {
FsPermission.createImmutable((short) 0x1ff));
final INodeFile file = new INodeFile(
MOCK_INODE_ID, FILE_NAME.getBytes(Charsets.UTF_8),
- perm, 1, 1, new BlockInfoContiguous[] {}, (short) 1,
+ perm, 1, 1, new BlockInfo[] {}, (short) 1,
DFS_BLOCK_SIZE_DEFAULT);
fsn.getFSDirectory().addINode(iip, file);
return fsn;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index b45d2f6..21b67a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -60,7 +60,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.IOUtils;
@@ -290,7 +290,7 @@ public class TestINodeFile {
iNodes[i] = new INodeFile(i, null, perm, 0L, 0L, null, replication,
preferredBlockSize);
iNodes[i].setLocalName(DFSUtil.string2Bytes(fileNamePrefix + i));
- BlockInfoContiguous newblock = new BlockInfoContiguous(replication);
+ BlockInfo newblock = new BlockInfo(replication);
iNodes[i].addBlock(newblock);
}
[10/50] [abbrv] hadoop git commit: HDFS-5033. Bad error message for
fs -put/copyFromLocal if user doesn't have permissions to read the source
(Darrell Taylor via aw)
Posted by zj...@apache.org.
HDFS-5033. Bad error message for fs -put/copyFromLocal if user doesn't have permissions to read the source (Darrell Taylor via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fdef78a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fdef78a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fdef78a
Branch: refs/heads/YARN-2928
Commit: 6fdef78a65c8c7e6fdf1bdfb05706fd867fc28e4
Parents: e06811e
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed May 27 16:51:34 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:53 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileUtil.java | 158 +++++-----
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/TestDFSShell.java | 297 ++++++++++++-------
3 files changed, 268 insertions(+), 190 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fdef78a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 91f00e1..5fd89c4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -65,7 +65,7 @@ public class FileUtil {
/**
* convert an array of FileStatus to an array of Path
- *
+ *
* @param stats
* an array of FileStatus objects
* @return an array of paths corresponding to the input
@@ -95,7 +95,7 @@ public class FileUtil {
else
return stat2Paths(stats);
}
-
+
/**
* Delete a directory and all its contents. If
* we return false, the directory may be partially-deleted.
@@ -110,7 +110,7 @@ public class FileUtil {
public static boolean fullyDelete(final File dir) {
return fullyDelete(dir, false);
}
-
+
/**
* Delete a directory and all its contents. If
* we return false, the directory may be partially-deleted.
@@ -127,7 +127,7 @@ public class FileUtil {
*/
public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
if (tryGrantPermissions) {
- // try to chmod +rwx the parent folder of the 'dir':
+ // try to chmod +rwx the parent folder of the 'dir':
File parent = dir.getParentFile();
grantPermissions(parent);
}
@@ -189,7 +189,7 @@ public class FileUtil {
}
return !ex;
}
-
+
/**
* Delete the contents of a directory, not the directory itself. If
* we return false, the directory may be partially-deleted.
@@ -199,19 +199,19 @@ public class FileUtil {
public static boolean fullyDeleteContents(final File dir) {
return fullyDeleteContents(dir, false);
}
-
+
/**
* Delete the contents of a directory, not the directory itself. If
* we return false, the directory may be partially-deleted.
* If dir is a symlink to a directory, all the contents of the actual
* directory pointed to by dir will be deleted.
- * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
+ * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
* and all the underlying directories before trying to delete their contents.
*/
public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
if (tryGrantPermissions) {
// to be able to list the dir and delete files from it
- // we must grant the dir rwx permissions:
+ // we must grant the dir rwx permissions:
grantPermissions(dir);
}
boolean deletionSucceeded = true;
@@ -246,14 +246,14 @@ public class FileUtil {
/**
* Recursively delete a directory.
- *
+ *
* @param fs {@link FileSystem} on which the path is present
- * @param dir directory to recursively delete
+ * @param dir directory to recursively delete
* @throws IOException
* @deprecated Use {@link FileSystem#delete(Path, boolean)}
*/
@Deprecated
- public static void fullyDelete(FileSystem fs, Path dir)
+ public static void fullyDelete(FileSystem fs, Path dir)
throws IOException {
fs.delete(dir, true);
}
@@ -262,9 +262,9 @@ public class FileUtil {
// If the destination is a subdirectory of the source, then
// generate exception
//
- private static void checkDependencies(FileSystem srcFS,
- Path src,
- FileSystem dstFS,
+ private static void checkDependencies(FileSystem srcFS,
+ Path src,
+ FileSystem dstFS,
Path dst)
throws IOException {
if (srcFS == dstFS) {
@@ -282,16 +282,16 @@ public class FileUtil {
}
/** Copy files between FileSystems. */
- public static boolean copy(FileSystem srcFS, Path src,
- FileSystem dstFS, Path dst,
+ public static boolean copy(FileSystem srcFS, Path src,
+ FileSystem dstFS, Path dst,
boolean deleteSource,
Configuration conf) throws IOException {
return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
}
- public static boolean copy(FileSystem srcFS, Path[] srcs,
+ public static boolean copy(FileSystem srcFS, Path[] srcs,
FileSystem dstFS, Path dst,
- boolean deleteSource,
+ boolean deleteSource,
boolean overwrite, Configuration conf)
throws IOException {
boolean gotException = false;
@@ -307,7 +307,7 @@ public class FileUtil {
"does not exist");
} else {
FileStatus sdst = dstFS.getFileStatus(dst);
- if (!sdst.isDirectory())
+ if (!sdst.isDirectory())
throw new IOException("copying multiple files, but last argument `" +
dst + "' is not a directory");
}
@@ -329,8 +329,8 @@ public class FileUtil {
}
/** Copy files between FileSystems. */
- public static boolean copy(FileSystem srcFS, Path src,
- FileSystem dstFS, Path dst,
+ public static boolean copy(FileSystem srcFS, Path src,
+ FileSystem dstFS, Path dst,
boolean deleteSource,
boolean overwrite,
Configuration conf) throws IOException {
@@ -375,21 +375,21 @@ public class FileUtil {
} else {
return true;
}
-
+
}
/** Copy all files in a directory to one output file (merge). */
- public static boolean copyMerge(FileSystem srcFS, Path srcDir,
- FileSystem dstFS, Path dstFile,
+ public static boolean copyMerge(FileSystem srcFS, Path srcDir,
+ FileSystem dstFS, Path dstFile,
boolean deleteSource,
Configuration conf, String addString) throws IOException {
dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
if (!srcFS.getFileStatus(srcDir).isDirectory())
return false;
-
+
OutputStream out = dstFS.create(dstFile);
-
+
try {
FileStatus contents[] = srcFS.listStatus(srcDir);
Arrays.sort(contents);
@@ -400,24 +400,24 @@ public class FileUtil {
IOUtils.copyBytes(in, out, conf, false);
if (addString!=null)
out.write(addString.getBytes("UTF-8"));
-
+
} finally {
in.close();
- }
+ }
}
}
} finally {
out.close();
}
-
+
if (deleteSource) {
return srcFS.delete(srcDir, true);
} else {
return true;
}
- }
-
+ }
+
/** Copy local files to a FileSystem. */
public static boolean copy(File src,
FileSystem dstFS, Path dst,
@@ -446,8 +446,12 @@ public class FileUtil {
IOUtils.closeStream( in );
throw e;
}
+ } else if (!src.canRead()) {
+ throw new IOException(src.toString() +
+ ": Permission denied");
+
} else {
- throw new IOException(src.toString() +
+ throw new IOException(src.toString() +
": No such file or directory");
}
if (deleteSource) {
@@ -458,7 +462,7 @@ public class FileUtil {
}
/** Copy FileSystem files to local files. */
- public static boolean copy(FileSystem srcFS, Path src,
+ public static boolean copy(FileSystem srcFS, Path src,
File dst, boolean deleteSource,
Configuration conf) throws IOException {
FileStatus filestatus = srcFS.getFileStatus(src);
@@ -516,7 +520,7 @@ public class FileUtil {
public static String makeShellPath(String filename) throws IOException {
return filename;
}
-
+
/**
* Convert a os-native filename to a path that works for the shell.
* @param file The filename to convert
@@ -530,12 +534,12 @@ public class FileUtil {
/**
* Convert a os-native filename to a path that works for the shell.
* @param file The filename to convert
- * @param makeCanonicalPath
+ * @param makeCanonicalPath
* Whether to make canonical path for the file passed
* @return The unix pathname
* @throws IOException on windows, there can be problems with the subprocess
*/
- public static String makeShellPath(File file, boolean makeCanonicalPath)
+ public static String makeShellPath(File file, boolean makeCanonicalPath)
throws IOException {
if (makeCanonicalPath) {
return makeShellPath(file.getCanonicalPath());
@@ -547,7 +551,7 @@ public class FileUtil {
/**
* Takes an input dir and returns the du on that local directory. Very basic
* implementation.
- *
+ *
* @param dir
* The input dir to get the disk space of this local dir
* @return The total disk space of the input local directory
@@ -576,7 +580,7 @@ public class FileUtil {
return size;
}
}
-
+
/**
* Given a File input it will unzip the file in a the unzip directory
* passed as the second parameter
@@ -596,9 +600,9 @@ public class FileUtil {
InputStream in = zipFile.getInputStream(entry);
try {
File file = new File(unzipDir, entry.getName());
- if (!file.getParentFile().mkdirs()) {
+ if (!file.getParentFile().mkdirs()) {
if (!file.getParentFile().isDirectory()) {
- throw new IOException("Mkdirs failed to create " +
+ throw new IOException("Mkdirs failed to create " +
file.getParentFile().toString());
}
}
@@ -625,10 +629,10 @@ public class FileUtil {
/**
* Given a Tar File as input it will untar the file in a the untar directory
* passed as the second parameter
- *
+ *
* This utility will untar ".tar" files and ".tar.gz","tgz" files.
- *
- * @param inFile The tar file as input.
+ *
+ * @param inFile The tar file as input.
* @param untarDir The untar directory where to untar the tar file.
* @throws IOException
*/
@@ -641,17 +645,17 @@ public class FileUtil {
boolean gzipped = inFile.toString().endsWith("gz");
if(Shell.WINDOWS) {
- // Tar is not native to Windows. Use simple Java based implementation for
+ // Tar is not native to Windows. Use simple Java based implementation for
// tests and simple tar archives
unTarUsingJava(inFile, untarDir, gzipped);
}
else {
- // spawn tar utility to untar archive for full fledged unix behavior such
+ // spawn tar utility to untar archive for full fledged unix behavior such
// as resolving symlinks in tar archives
unTarUsingTar(inFile, untarDir, gzipped);
}
}
-
+
private static void unTarUsingTar(File inFile, File untarDir,
boolean gzipped) throws IOException {
StringBuffer untarCommand = new StringBuffer();
@@ -659,9 +663,9 @@ public class FileUtil {
untarCommand.append(" gzip -dc '");
untarCommand.append(FileUtil.makeShellPath(inFile));
untarCommand.append("' | (");
- }
+ }
untarCommand.append("cd '");
- untarCommand.append(FileUtil.makeShellPath(untarDir));
+ untarCommand.append(FileUtil.makeShellPath(untarDir));
untarCommand.append("' ; ");
untarCommand.append("tar -xf ");
@@ -675,11 +679,11 @@ public class FileUtil {
shexec.execute();
int exitcode = shexec.getExitCode();
if (exitcode != 0) {
- throw new IOException("Error untarring file " + inFile +
+ throw new IOException("Error untarring file " + inFile +
". Tar process exited with exit code " + exitcode);
}
}
-
+
private static void unTarUsingJava(File inFile, File untarDir,
boolean gzipped) throws IOException {
InputStream inputStream = null;
@@ -702,7 +706,7 @@ public class FileUtil {
IOUtils.cleanup(LOG, tis, inputStream);
}
}
-
+
private static void unpackEntries(TarArchiveInputStream tis,
TarArchiveEntry entry, File outputDir) throws IOException {
if (entry.isDirectory()) {
@@ -739,14 +743,14 @@ public class FileUtil {
outputStream.flush();
outputStream.close();
}
-
+
/**
* Class for creating hardlinks.
* Supports Unix, WindXP.
* @deprecated Use {@link org.apache.hadoop.fs.HardLink}
*/
@Deprecated
- public static class HardLink extends org.apache.hadoop.fs.HardLink {
+ public static class HardLink extends org.apache.hadoop.fs.HardLink {
// This is a stub to assist with coordinated change between
// COMMON and HDFS projects. It will be removed after the
// corresponding change is committed to HDFS.
@@ -759,7 +763,7 @@ public class FileUtil {
* setting, we will log a warning. The return code in this
* case is 2.
*
- * @param target the target for symlink
+ * @param target the target for symlink
* @param linkname the symlink
* @return 0 on success
*/
@@ -873,7 +877,7 @@ public class FileUtil {
shExec.execute();
}catch(IOException e) {
if(LOG.isDebugEnabled()) {
- LOG.debug("Error while changing permission : " + filename
+ LOG.debug("Error while changing permission : " + filename
+" Exception: " + StringUtils.stringifyException(e));
}
}
@@ -1041,9 +1045,9 @@ public class FileUtil {
execSetPermission(f, permission);
return;
}
-
+
boolean rv = true;
-
+
// read perms
rv = f.setReadable(group.implies(FsAction.READ), false);
checkReturnValue(rv, f, permission);
@@ -1069,17 +1073,17 @@ public class FileUtil {
}
}
- private static void checkReturnValue(boolean rv, File p,
+ private static void checkReturnValue(boolean rv, File p,
FsPermission permission
) throws IOException {
if (!rv) {
- throw new IOException("Failed to set permissions of path: " + p +
- " to " +
+ throw new IOException("Failed to set permissions of path: " + p +
+ " to " +
String.format("%04o", permission.toShort()));
}
}
-
- private static void execSetPermission(File f,
+
+ private static void execSetPermission(File f,
FsPermission permission
) throws IOException {
if (NativeIO.isAvailable()) {
@@ -1089,7 +1093,7 @@ public class FileUtil {
String.format("%04o", permission.toShort()), false));
}
}
-
+
static String execCommand(File f, String... cmd) throws IOException {
String[] args = new String[cmd.length + 1];
System.arraycopy(cmd, 0, args, 0, cmd.length);
@@ -1147,12 +1151,12 @@ public class FileUtil {
}
}
}
-
+
/**
- * A wrapper for {@link File#listFiles()}. This java.io API returns null
+ * A wrapper for {@link File#listFiles()}. This java.io API returns null
* when a dir is not a directory or for any I/O error. Instead of having
* null check everywhere File#listFiles() is used, we will add utility API
- * to get around this problem. For the majority of cases where we prefer
+ * to get around this problem. For the majority of cases where we prefer
* an IOException to be thrown.
* @param dir directory for which listing should be performed
* @return list of files or empty list
@@ -1165,13 +1169,13 @@ public class FileUtil {
+ dir.toString());
}
return files;
- }
-
+ }
+
/**
- * A wrapper for {@link File#list()}. This java.io API returns null
+ * A wrapper for {@link File#list()}. This java.io API returns null
* when a dir is not a directory or for any I/O error. Instead of having
* null check everywhere File#list() is used, we will add utility API
- * to get around this problem. For the majority of cases where we prefer
+ * to get around this problem. For the majority of cases where we prefer
* an IOException to be thrown.
* @param dir directory for which listing should be performed
* @return list of file names or empty string list
@@ -1184,35 +1188,35 @@ public class FileUtil {
+ dir.toString());
}
return fileNames;
- }
-
+ }
+
public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
Map<String, String> callerEnv) throws IOException {
return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv);
}
-
+
/**
* Create a jar file at the given path, containing a manifest with a classpath
* that references all specified entries.
- *
+ *
* Some platforms may have an upper limit on command line length. For example,
* the maximum command line length on Windows is 8191 characters, but the
* length of the classpath may exceed this. To work around this limitation,
* use this method to create a small intermediate jar with a manifest that
* contains the full classpath. It returns the absolute path to the new jar,
* which the caller may set as the classpath for a new process.
- *
+ *
* Environment variable evaluation is not supported within a jar manifest, so
* this method expands environment variables before inserting classpath entries
* to the manifest. The method parses environment variables according to
* platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows,
* environment variables are case-insensitive. For example, %VAR% and %var%
* evaluate to the same value.
- *
+ *
* Specifying the classpath in a jar manifest does not support wildcards, so
* this method expands wildcards internally. Any classpath entry that ends
* with * is translated to all files at that path with extension .jar or .JAR.
- *
+ *
* @param inputClassPath String input classpath to bundle into the jar manifest
* @param pwd Path to working directory to save jar
* @param targetDir path to where the jar execution will have its working dir
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fdef78a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 948a516..439b53a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -160,6 +160,9 @@ Trunk (Unreleased)
HDFS-6353. Check and make checkpoint before stopping the NameNode. (jing9)
+ HDFS-5033. Bad error message for fs -put/copyFromLocal if user
+ doesn't have permissions to read the source (Darrell Taylor via aw)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fdef78a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index c1bf771..2df31c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -155,13 +155,13 @@ public class TestDFSShell {
assertTrue(f1.exists());
assertTrue(f1.isFile());
assertEquals(0L, f1.length());
-
+
//copy to remote
final Path root = mkdir(dfs, new Path("/test/zeroSizeFile"));
final Path remotef = new Path(root, "dst");
show("copy local " + f1 + " to remote " + remotef);
dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), remotef);
-
+
//getBlockSize() should not throw exception
show("Block size = " + dfs.getFileStatus(remotef).getBlockSize());
@@ -172,7 +172,7 @@ public class TestDFSShell {
assertTrue(f2.exists());
assertTrue(f2.isFile());
assertEquals(0L, f2.length());
-
+
f1.delete();
f2.delete();
} finally {
@@ -180,13 +180,13 @@ public class TestDFSShell {
cluster.shutdown();
}
}
-
+
@Test (timeout = 30000)
public void testRecursiveRm() throws IOException {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
FileSystem fs = cluster.getFileSystem();
- assertTrue("Not a HDFS: " + fs.getUri(),
+ assertTrue("Not a HDFS: " + fs.getUri(),
fs instanceof DistributedFileSystem);
try {
fs.mkdirs(new Path(new Path("parent"), "child"));
@@ -201,12 +201,12 @@ public class TestDFSShell {
} catch(IOException e) {
assert(false);
}
- } finally {
+ } finally {
try { fs.close();}catch(IOException e){};
cluster.shutdown();
}
}
-
+
@Test (timeout = 30000)
public void testDu() throws IOException {
int replication = 2;
@@ -220,7 +220,7 @@ public class TestDFSShell {
System.setOut(psOut);
FsShell shell = new FsShell();
shell.setConf(conf);
-
+
try {
Path myPath = new Path("/test/dir");
assertTrue(fs.mkdirs(myPath));
@@ -235,7 +235,7 @@ public class TestDFSShell {
Long myFileDiskUsed = myFileLength * replication;
Long myFile2Length = fs.getFileStatus(myFile2).getLen();
Long myFile2DiskUsed = myFile2Length * replication;
-
+
String[] args = new String[2];
args[0] = "-du";
args[1] = "/test/dir";
@@ -254,7 +254,7 @@ public class TestDFSShell {
assertThat(returnString, containsString(myFileDiskUsed.toString()));
assertThat(returnString, containsString(myFile2Length.toString()));
assertThat(returnString, containsString(myFile2DiskUsed.toString()));
-
+
// Check that -du -s reports the state of the snapshot
String snapshotName = "ss1";
Path snapshotPath = new Path(myPath, ".snapshot/" + snapshotName);
@@ -306,7 +306,7 @@ public class TestDFSShell {
System.setOut(psBackup);
cluster.shutdown();
}
-
+
}
@Test (timeout = 30000)
@@ -321,15 +321,15 @@ public class TestDFSShell {
try {
// remove left over crc files:
new File(TEST_ROOT_DIR, ".f1.crc").delete();
- new File(TEST_ROOT_DIR, ".f2.crc").delete();
+ new File(TEST_ROOT_DIR, ".f2.crc").delete();
final File f1 = createLocalFile(new File(TEST_ROOT_DIR, "f1"));
final File f2 = createLocalFile(new File(TEST_ROOT_DIR, "f2"));
-
+
final Path root = mkdir(dfs, new Path("/test/put"));
final Path dst = new Path(root, "dst");
-
+
show("begin");
-
+
final Thread copy2ndFileThread = new Thread() {
@Override
public void run() {
@@ -344,13 +344,13 @@ public class TestDFSShell {
assertTrue(false);
}
};
-
+
//use SecurityManager to pause the copying of f1 and begin copying f2
SecurityManager sm = System.getSecurityManager();
System.out.println("SecurityManager = " + sm);
System.setSecurityManager(new SecurityManager() {
private boolean firstTime = true;
-
+
@Override
public void checkPermission(Permission perm) {
if (firstTime) {
@@ -359,7 +359,7 @@ public class TestDFSShell {
String s = "" + Arrays.asList(t.getStackTrace());
if (s.contains("FileUtil.copyContent")) {
//pause at FileUtil.copyContent
-
+
firstTime = false;
copy2ndFileThread.start();
try {Thread.sleep(5000);} catch (InterruptedException e) {}
@@ -371,7 +371,7 @@ public class TestDFSShell {
show("copy local " + f1 + " to remote " + dst);
dfs.copyFromLocalFile(false, false, new Path(f1.getPath()), dst);
show("done");
-
+
try {copy2ndFileThread.join();} catch (InterruptedException e) { }
System.setSecurityManager(sm);
@@ -381,8 +381,8 @@ public class TestDFSShell {
srcs[0] = new Path(f1.getPath());
srcs[1] = new Path(f2.getPath());
dfs.copyFromLocalFile(false, false, srcs, destmultiple);
- srcs[0] = new Path(destmultiple,"f1");
- srcs[1] = new Path(destmultiple,"f2");
+ srcs[0] = new Path(destmultiple,"f1");
+ srcs[1] = new Path(destmultiple,"f2");
assertTrue(dfs.exists(srcs[0]));
assertTrue(dfs.exists(srcs[1]));
@@ -492,7 +492,7 @@ public class TestDFSShell {
ret = ToolRunner.run(shell, argv);
returned = out.toString();
assertEquals(" -mkdir returned 1 ", 1, ret);
- assertTrue(" -mkdir returned File exists",
+ assertTrue(" -mkdir returned File exists",
(returned.lastIndexOf("File exists") != -1));
Path testFile = new Path("/testfile");
OutputStream outtmp = srcFs.create(testFile);
@@ -520,7 +520,7 @@ public class TestDFSShell {
argv[2] = "/testfiletest";
ret = ToolRunner.run(shell, argv);
returned = out.toString();
- assertTrue("no output from rename",
+ assertTrue("no output from rename",
(returned.lastIndexOf("Renamed") == -1));
out.reset();
argv[0] = "-mv";
@@ -557,7 +557,7 @@ public class TestDFSShell {
}
}
}
-
+
@Test (timeout = 30000)
public void testURIPaths() throws Exception {
Configuration srcConf = new HdfsConfiguration();
@@ -580,14 +580,14 @@ public class TestDFSShell {
argv[1] = dstFs.getUri().toString() + "/";
int ret = ToolRunner.run(shell, argv);
assertEquals("ls works on remote uri ", 0, ret);
- //check for rm -r
+ //check for rm -r
dstFs.mkdirs(new Path("/hadoopdir"));
argv = new String[2];
argv[0] = "-rmr";
argv[1] = dstFs.getUri().toString() + "/hadoopdir";
ret = ToolRunner.run(shell, argv);
assertEquals("-rmr works on remote uri " + argv[1], 0, ret);
- //check du
+ //check du
argv[0] = "-du";
argv[1] = dstFs.getUri().toString() + "/";
ret = ToolRunner.run(shell, argv);
@@ -601,14 +601,14 @@ public class TestDFSShell {
argv[2] = dstFs.getUri().toString() + "/furi";
ret = ToolRunner.run(shell, argv);
assertEquals(" put is working ", 0, ret);
- //check cp
+ //check cp
argv[0] = "-cp";
argv[1] = dstFs.getUri().toString() + "/furi";
argv[2] = srcFs.getUri().toString() + "/furi";
ret = ToolRunner.run(shell, argv);
assertEquals(" cp is working ", 0, ret);
assertTrue(srcFs.exists(new Path("/furi")));
- //check cat
+ //check cat
argv = new String[2];
argv[0] = "-cat";
argv[1] = dstFs.getUri().toString() + "/furi";
@@ -626,7 +626,7 @@ public class TestDFSShell {
confirmOwner(null, "herbivores", dstFs, parent, path);
runCmd(shell, "-chown", "-R", ":reptiles", dstFs.getUri().toString() + "/");
confirmOwner(null, "reptiles", dstFs, root, parent, path);
- //check if default hdfs:/// works
+ //check if default hdfs:/// works
argv[0] = "-cat";
argv[1] = "hdfs:///furi";
ret = ToolRunner.run(shell, argv);
@@ -757,7 +757,7 @@ public class TestDFSShell {
assertTrue("Output doesn't match input",
Arrays.equals(writebytes, out.toByteArray()));
out.reset();
-
+
// Test a plain text.
OutputStream pout = fs.create(new Path(root, "file.txt"));
writebytes = "bar".getBytes();
@@ -805,8 +805,8 @@ public class TestDFSShell {
}
File localroot = new File(TEST_ROOT_DIR, "copyToLocal");
- File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2");
-
+ File localroot2 = new File(TEST_ROOT_DIR, "copyToLocal2");
+
File f1 = new File(localroot, "f1");
assertTrue("Copying failed.", f1.isFile());
@@ -821,9 +821,9 @@ public class TestDFSShell {
File f4 = new File(sub, "f4");
assertTrue("Copying failed.", f4.isFile());
-
+
File f5 = new File(localroot2, "f1");
- assertTrue("Copying failed.", f5.isFile());
+ assertTrue("Copying failed.", f5.isFile());
f1.delete();
f2.delete();
@@ -836,12 +836,12 @@ public class TestDFSShell {
// destination files
{
String[] args = {"-copyToLocal", "nosuchfile", TEST_ROOT_DIR};
- try {
+ try {
assertEquals(1, shell.run(args));
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
e.getLocalizedMessage());
- }
+ }
File f6 = new File(TEST_ROOT_DIR, "nosuchfile");
assertTrue(!f6.exists());
}
@@ -867,7 +867,7 @@ public class TestDFSShell {
String path = "/test/" + name;
Path root = mkdir(fs, new Path(path));
Path sub = mkdir(fs, new Path(root, "sub"));
- Path root2 = mkdir(fs, new Path(path + "2"));
+ Path root2 = mkdir(fs, new Path(path + "2"));
writeFile(fs, new Path(root, "f1"));
writeFile(fs, new Path(root, "f2"));
@@ -900,7 +900,7 @@ public class TestDFSShell {
localpath = localpath.makeQualified(localfs.getUri(),
localfs.getWorkingDirectory());
localfs.mkdirs(localpath);
-
+
final String localstr = localpath.toString();
System.out.println("localstr=" + localstr);
runCount(localstr, 1, 0, shell);
@@ -915,7 +915,7 @@ public class TestDFSShell {
}
private static void runCount(String path, long dirs, long files, FsShell shell
) throws IOException {
- ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
PrintStream out = new PrintStream(bytes);
PrintStream oldOut = System.out;
System.setOut(out);
@@ -956,15 +956,15 @@ public class TestDFSShell {
throw new IOException(StringUtils.stringifyException(e));
}
}
-
+
/**
* Test chmod.
*/
- void testChmod(Configuration conf, FileSystem fs, String chmodDir)
+ void testChmod(Configuration conf, FileSystem fs, String chmodDir)
throws IOException {
FsShell shell = new FsShell();
shell.setConf(conf);
-
+
try {
//first make dir
Path dir = new Path(chmodDir);
@@ -1039,8 +1039,8 @@ public class TestDFSShell {
LOG.info("Permission change result: " + result);
assertEquals(expected, result);
}
-
- private void confirmOwner(String owner, String group,
+
+ private void confirmOwner(String owner, String group,
FileSystem fs, Path... paths) throws IOException {
for(Path path : paths) {
if (owner != null) {
@@ -1051,68 +1051,68 @@ public class TestDFSShell {
}
}
}
-
+
@Test (timeout = 30000)
public void testFilePermissions() throws IOException {
Configuration conf = new HdfsConfiguration();
-
+
//test chmod on local fs
FileSystem fs = FileSystem.getLocal(conf);
- testChmod(conf, fs,
+ testChmod(conf, fs,
(new File(TEST_ROOT_DIR, "chmodTest")).getAbsolutePath());
-
+
conf.set(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, "true");
-
+
//test chmod on DFS
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
fs = cluster.getFileSystem();
testChmod(conf, fs, "/tmp/chmodTest");
-
+
// test chown and chgrp on DFS:
-
+
FsShell shell = new FsShell();
shell.setConf(conf);
fs = cluster.getFileSystem();
-
+
/* For dfs, I am the super user and I can change owner of any file to
* anything. "-R" option is already tested by chmod test above.
*/
-
+
String file = "/tmp/chownTest";
Path path = new Path(file);
Path parent = new Path("/tmp");
Path root = new Path("/");
TestDFSShell.writeFile(fs, path);
-
+
runCmd(shell, "-chgrp", "-R", "herbivores", "/*", "unknownFile*");
confirmOwner(null, "herbivores", fs, parent, path);
-
+
runCmd(shell, "-chgrp", "mammals", file);
confirmOwner(null, "mammals", fs, path);
-
+
runCmd(shell, "-chown", "-R", ":reptiles", "/");
confirmOwner(null, "reptiles", fs, root, parent, path);
-
+
runCmd(shell, "-chown", "python:", "/nonExistentFile", file);
confirmOwner("python", "reptiles", fs, path);
runCmd(shell, "-chown", "-R", "hadoop:toys", "unknownFile", "/");
confirmOwner("hadoop", "toys", fs, root, parent, path);
-
+
// Test different characters in names
runCmd(shell, "-chown", "hdfs.user", file);
confirmOwner("hdfs.user", null, fs, path);
-
+
runCmd(shell, "-chown", "_Hdfs.User-10:_hadoop.users--", file);
confirmOwner("_Hdfs.User-10", "_hadoop.users--", fs, path);
-
+
runCmd(shell, "-chown", "hdfs/hadoop-core@apache.org:asf-projects", file);
confirmOwner("hdfs/hadoop-core@apache.org", "asf-projects", fs, path);
-
+
runCmd(shell, "-chgrp", "hadoop-core@apache.org/100", file);
confirmOwner(null, "hadoop-core@apache.org/100", fs, path);
-
+
cluster.shutdown();
}
/**
@@ -1142,7 +1142,7 @@ public class TestDFSShell {
Path myFile = new Path("/test/mkdirs/myFile");
writeFile(fileSys, myFile);
assertTrue(fileSys.exists(myFile));
- Path myFile2 = new Path("/test/mkdirs/myFile2");
+ Path myFile2 = new Path("/test/mkdirs/myFile2");
writeFile(fileSys, myFile2);
assertTrue(fileSys.exists(myFile2));
@@ -1156,7 +1156,7 @@ public class TestDFSShell {
val = shell.run(args);
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
- e.getLocalizedMessage());
+ e.getLocalizedMessage());
}
assertTrue(val == 0);
assertFalse(fileSys.exists(myFile));
@@ -1174,13 +1174,13 @@ public class TestDFSShell {
String[] args = new String[3];
args[0] = "-cat";
args[1] = "/test/mkdirs/myFile";
- args[2] = "/test/mkdirs/myFile2";
+ args[2] = "/test/mkdirs/myFile2";
int val = -1;
try {
val = shell.run(args);
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run: " +
- StringUtils.stringifyException(e));
+ StringUtils.stringifyException(e));
}
assertTrue(val == 0);
}
@@ -1196,7 +1196,7 @@ public class TestDFSShell {
val = shell.run(args);
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
- e.getLocalizedMessage());
+ e.getLocalizedMessage());
}
assertTrue(val != 0);
}
@@ -1211,7 +1211,7 @@ public class TestDFSShell {
val = shell.run(args);
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
- e.getLocalizedMessage());
+ e.getLocalizedMessage());
}
assertTrue(val != 0);
}
@@ -1226,7 +1226,7 @@ public class TestDFSShell {
val = shell.run(args);
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
- e.getLocalizedMessage());
+ e.getLocalizedMessage());
}
assertTrue(val == 0);
}
@@ -1480,7 +1480,7 @@ public class TestDFSShell {
for(Block b : e.getValue()) {
files.add(DataNodeTestUtils.getFile(dn, poolId, b.getBlockId()));
}
- }
+ }
}
return files;
}
@@ -1493,7 +1493,7 @@ public class TestDFSShell {
PrintWriter out = new PrintWriter(f);
out.print(content);
out.flush();
- out.close();
+ out.close();
}
}
@@ -1503,7 +1503,7 @@ public class TestDFSShell {
@Test (timeout = 30000)
public void testRemoteException() throws Exception {
- UserGroupInformation tmpUGI =
+ UserGroupInformation tmpUGI =
UserGroupInformation.createUserForTesting("tmpname", new String[] {"mygroup"});
MiniDFSCluster dfs = null;
PrintStream bak = null;
@@ -1515,7 +1515,7 @@ public class TestDFSShell {
fs.mkdirs(p);
fs.setPermission(p, new FsPermission((short)0700));
bak = System.err;
-
+
tmpUGI.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
@@ -1529,9 +1529,9 @@ public class TestDFSShell {
int ret = ToolRunner.run(fshell, args);
assertEquals("returned should be 1", 1, ret);
String str = out.toString();
- assertTrue("permission denied printed",
+ assertTrue("permission denied printed",
str.indexOf("Permission denied") != -1);
- out.reset();
+ out.reset();
return null;
}
});
@@ -1544,7 +1544,7 @@ public class TestDFSShell {
}
}
}
-
+
@Test (timeout = 30000)
public void testGet() throws IOException {
GenericTestUtils.setLogLevel(FSInputChecker.LOG, Level.ALL);
@@ -1563,7 +1563,7 @@ public class TestDFSShell {
String dst = new File(TEST_ROOT_DIR, fname + ++count)
.getAbsolutePath();
String[] args = new String[options.length + 3];
- args[0] = "-get";
+ args[0] = "-get";
args[args.length - 2] = remotef.toString();
args[args.length - 1] = dst;
for(int i = 0; i < options.length; i++) {
@@ -1574,9 +1574,9 @@ public class TestDFSShell {
try {
assertEquals(exitcode, shell.run(args));
} catch (Exception e) {
- assertTrue(StringUtils.stringifyException(e), false);
+ assertTrue(StringUtils.stringifyException(e), false);
}
- return exitcode == 0? DFSTestUtil.readFile(new File(dst)): null;
+ return exitcode == 0? DFSTestUtil.readFile(new File(dst)): null;
}
};
@@ -1645,9 +1645,9 @@ public class TestDFSShell {
try {
final String root = createTree(dfs, "lsr");
dfs.mkdirs(new Path(root, "zzz"));
-
+
runLsr(new FsShell(conf), root, 0);
-
+
final Path sub = new Path(root, "sub");
dfs.setPermission(sub, new FsPermission((short)0));
@@ -1670,7 +1670,7 @@ public class TestDFSShell {
private static String runLsr(final FsShell shell, String root, int returnvalue
) throws Exception {
System.out.println("root=" + root + ", returnvalue=" + returnvalue);
- final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
final PrintStream out = new PrintStream(bytes);
final PrintStream oldOut = System.out;
final PrintStream oldErr = System.err;
@@ -1688,7 +1688,7 @@ public class TestDFSShell {
System.out.println("results:\n" + results);
return results;
}
-
+
/**
* default setting is file:// which is not a DFS
* so DFSAdmin should throw and catch InvalidArgumentException
@@ -1703,7 +1703,7 @@ public class TestDFSShell {
int res = admin.run(new String[] {"-refreshNodes"});
assertEquals("expected to fail -1", res , -1);
}
-
+
// Preserve Copy Option is -ptopxa (timestamps, ownership, permission, XATTR,
// ACLs)
@Test (timeout = 120000)
@@ -1737,15 +1737,15 @@ public class TestDFSShell {
final String owner = status.getOwner();
final String group = status.getGroup();
final FsPermission perm = status.getPermission();
-
+
fs.setXAttr(src, USER_A1, USER_A1_VALUE);
fs.setXAttr(src, TRUSTED_A1, TRUSTED_A1_VALUE);
-
+
shell = new FsShell(conf);
-
+
// -p
Path target1 = new Path(hdfsTestDir, "targetfile1");
- String[] argv = new String[] { "-cp", "-p", src.toUri().toString(),
+ String[] argv = new String[] { "-cp", "-p", src.toUri().toString(),
target1.toUri().toString() };
int ret = ToolRunner.run(shell, argv);
assertEquals("cp -p is not working", SUCCESS, ret);
@@ -1764,7 +1764,7 @@ public class TestDFSShell {
// -ptop
Path target2 = new Path(hdfsTestDir, "targetfile2");
- argv = new String[] { "-cp", "-ptop", src.toUri().toString(),
+ argv = new String[] { "-cp", "-ptop", src.toUri().toString(),
target2.toUri().toString() };
ret = ToolRunner.run(shell, argv);
assertEquals("cp -ptop is not working", SUCCESS, ret);
@@ -1783,7 +1783,7 @@ public class TestDFSShell {
// -ptopx
Path target3 = new Path(hdfsTestDir, "targetfile3");
- argv = new String[] { "-cp", "-ptopx", src.toUri().toString(),
+ argv = new String[] { "-cp", "-ptopx", src.toUri().toString(),
target3.toUri().toString() };
ret = ToolRunner.run(shell, argv);
assertEquals("cp -ptopx is not working", SUCCESS, ret);
@@ -2317,6 +2317,77 @@ public class TestDFSShell {
}
}
+ /* [refs HDFS-5033]
+ *
+ * return a "Permission Denied" message instead of "No such file or Directory"
+ * when trying to put/copyFromLocal a file that doesn't have read access
+ *
+ */
+ @Test (timeout = 30000)
+ public void testCopyFromLocalWithPermissionDenied() throws Exception {
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+ .format(true).build();
+ FsShell shell = null;
+ FileSystem fs = null;
+ PrintStream bak = null;
+
+ final File localFile = new File(TEST_ROOT_DIR, "testFileWithNoReadPermissions");
+ final String localfilepath = new Path(localFile.getAbsolutePath()).toUri().toString();
+ final String testdir = "/tmp/TestDFSShell-CopyFromLocalWithPermissionDenied-"
+ + counter.getAndIncrement();
+ final Path hdfsTestDir = new Path(testdir);
+ try {
+ fs = cluster.getFileSystem();
+ fs.mkdirs(hdfsTestDir);
+ localFile.createNewFile();
+ localFile.setReadable(false);
+ writeFile(fs, new Path(testdir, "testFileForPut"));
+ shell = new FsShell();
+
+ // capture system error messages, snarfed from testErrOutPut()
+ bak = System.err;
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ PrintStream tmp = new PrintStream(out);
+ System.setErr(tmp);
+
+ // Tests for put
+ String[] argv = new String[] { "-put", localfilepath, testdir };
+ int res = ToolRunner.run(shell, argv);
+ assertEquals("put is working", ERROR, res);
+ String returned = out.toString();
+ assertTrue(" outputs Permission denied error message",
+ (returned.lastIndexOf("Permission denied") != -1));
+
+ // Tests for copyFromLocal
+ argv = new String[] { "-copyFromLocal", localfilepath, testdir };
+ res = ToolRunner.run(shell, argv);
+ assertEquals("copyFromLocal -f is working", ERROR, res);
+ returned = out.toString();
+ assertTrue(" outputs Permission denied error message",
+ (returned.lastIndexOf("Permission denied") != -1));
+
+ } finally {
+ if (bak != null) {
+ System.setErr(bak);
+ }
+
+ if (null != shell)
+ shell.close();
+
+ if (localFile.exists())
+ localFile.delete();
+
+ if (null != fs) {
+ fs.delete(hdfsTestDir, true);
+ fs.close();
+ }
+ cluster.shutdown();
+ }
+ }
+
+
+
// setrep for file and directory.
@Test (timeout = 30000)
public void testSetrep() throws Exception {
@@ -2506,7 +2577,7 @@ public class TestDFSShell {
cluster.shutdown();
}
}
-
+
@Test (timeout = 30000)
public void testSetXAttrPermission() throws Exception {
UserGroupInformation user = UserGroupInformation.
@@ -2517,16 +2588,16 @@ public class TestDFSShell {
final Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
-
+
FileSystem fs = cluster.getFileSystem();
Path p = new Path("/foo");
fs.mkdirs(p);
bak = System.err;
-
+
final FsShell fshell = new FsShell(conf);
final ByteArrayOutputStream out = new ByteArrayOutputStream();
System.setErr(new PrintStream(out));
-
+
// No permission to write xattr
fs.setPermission(p, new FsPermission((short) 0700));
user.doAs(new PrivilegedExceptionAction<Object>() {
@@ -2536,18 +2607,18 @@ public class TestDFSShell {
"-setfattr", "-n", "user.a1", "-v", "1234", "/foo"});
assertEquals("Returned should be 1", 1, ret);
String str = out.toString();
- assertTrue("Permission denied printed",
+ assertTrue("Permission denied printed",
str.indexOf("Permission denied") != -1);
out.reset();
return null;
}
});
-
+
int ret = ToolRunner.run(fshell, new String[]{
"-setfattr", "-n", "user.a1", "-v", "1234", "/foo"});
assertEquals("Returned should be 0", 0, ret);
out.reset();
-
+
// No permission to read and remove
fs.setPermission(p, new FsPermission((short) 0750));
user.doAs(new PrivilegedExceptionAction<Object>() {
@@ -2560,7 +2631,7 @@ public class TestDFSShell {
String str = out.toString();
assertTrue("Permission denied printed",
str.indexOf("Permission denied") != -1);
- out.reset();
+ out.reset();
// Remove
ret = ToolRunner.run(fshell, new String[]{
"-setfattr", "-x", "user.a1", "/foo"});
@@ -2568,7 +2639,7 @@ public class TestDFSShell {
str = out.toString();
assertTrue("Permission denied printed",
str.indexOf("Permission denied") != -1);
- out.reset();
+ out.reset();
return null;
}
});
@@ -2683,28 +2754,28 @@ public class TestDFSShell {
}
/**
- *
+ *
* Test to make sure that user namespace xattrs can be set only if path has
* access and for sticky directorries, only owner/privileged user can write.
* Trusted namespace xattrs can be set only with privileged users.
- *
+ *
* As user1: Create a directory (/foo) as user1, chown it to user1 (and
* user1's group), grant rwx to "other".
- *
+ *
* As user2: Set an xattr (should pass with path access).
- *
+ *
* As user1: Set an xattr (should pass).
- *
+ *
* As user2: Read the xattr (should pass). Remove the xattr (should pass with
* path access).
- *
+ *
* As user1: Read the xattr (should pass). Remove the xattr (should pass).
- *
+ *
* As user1: Change permissions only to owner
- *
+ *
* As User2: Set an Xattr (Should fail set with no path access) Remove an
* Xattr (Should fail with no path access)
- *
+ *
* As SuperUser: Set an Xattr with Trusted (Should pass)
*/
@Test (timeout = 30000)
@@ -2742,7 +2813,7 @@ public class TestDFSShell {
return null;
}
});
-
+
//Test 2. Give access to others
user1.doAs(new PrivilegedExceptionAction<Object>() {
@Override
@@ -2809,7 +2880,7 @@ public class TestDFSShell {
return null;
}
});
-
+
// Test 7. Change permission to have path access only to owner(user1)
user1.doAs(new PrivilegedExceptionAction<Object>() {
@Override
@@ -2822,7 +2893,7 @@ public class TestDFSShell {
return null;
}
});
-
+
// Test 8. There should be no permissions to set for
// the non-owning user with no path access.
user2.doAs(new PrivilegedExceptionAction<Object>() {
@@ -2839,7 +2910,7 @@ public class TestDFSShell {
return null;
}
});
-
+
// Test 9. There should be no permissions to remove for
// the non-owning user with no path access.
user2.doAs(new PrivilegedExceptionAction<Object>() {
@@ -2856,7 +2927,7 @@ public class TestDFSShell {
return null;
}
});
-
+
// Test 10. Superuser should be allowed to set with trusted namespace
SUPERUSER.doAs(new PrivilegedExceptionAction<Object>() {
@Override
[06/50] [abbrv] hadoop git commit: YARN-2355. MAX_APP_ATTEMPTS_ENV
may no longer be a useful env var for a container (Darrell Taylor via aw)
Posted by zj...@apache.org.
YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var for a container (Darrell Taylor via aw)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e06811e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e06811e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e06811e6
Branch: refs/heads/YARN-2928
Commit: e06811e689447cb30418ae66f1e39fe9ff2ecbb3
Parents: 0cd01a1
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed May 27 16:40:56 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:32 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/api/ApplicationConstants.java | 50 +++++++++-----------
.../resourcemanager/amlauncher/AMLauncher.java | 29 +++++-------
.../TestApplicationMasterLauncher.java | 7 ++-
4 files changed, 41 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c8dbb08..54b33bc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -136,6 +136,9 @@ Trunk - Unreleased
YARN-2428. LCE default banned user list should have yarn (Varun
Saxena via aw)
+ YARN-2355. MAX_APP_ATTEMPTS_ENV may no longer be a useful env var
+ for a container (Darrell Taylor via aw)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index 42464da..d75297a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -28,13 +28,13 @@ import org.apache.hadoop.util.Shell;
/**
* This is the API for the applications comprising of constants that YARN sets
* up for the applications and the containers.
- *
+ *
* TODO: Investigate the semantics and security of each cross-boundary refs.
*/
@Public
@Evolving
public interface ApplicationConstants {
-
+
/**
* The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
* only
@@ -48,11 +48,11 @@ public interface ApplicationConstants {
UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
/**
- * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
+ * The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
* ApplicationMaster's environment only. This states that for all non-relative
* web URLs in the app masters web UI what base should they have.
*/
- public static final String APPLICATION_WEB_PROXY_BASE_ENV =
+ public static final String APPLICATION_WEB_PROXY_BASE_ENV =
"APPLICATION_WEB_PROXY_BASE";
/**
@@ -99,15 +99,9 @@ public interface ApplicationConstants {
public static final String STDOUT = "stdout";
/**
- * The environment variable for MAX_APP_ATTEMPTS. Set in AppMaster environment
- * only
- */
- public static final String MAX_APP_ATTEMPTS_ENV = "MAX_APP_ATTEMPTS";
-
- /**
* Environment for Applications.
- *
- * Some of the environment variables for applications are <em>final</em>
+ *
+ * Some of the environment variables for applications are <em>final</em>
* i.e. they cannot be modified by the applications.
*/
public enum Environment {
@@ -116,76 +110,76 @@ public interface ApplicationConstants {
* Final, non-modifiable.
*/
USER("USER"),
-
+
/**
* $LOGNAME
* Final, non-modifiable.
*/
LOGNAME("LOGNAME"),
-
+
/**
* $HOME
* Final, non-modifiable.
*/
HOME("HOME"),
-
+
/**
* $PWD
* Final, non-modifiable.
*/
PWD("PWD"),
-
+
/**
* $PATH
*/
PATH("PATH"),
-
+
/**
* $SHELL
*/
SHELL("SHELL"),
-
+
/**
* $JAVA_HOME
*/
JAVA_HOME("JAVA_HOME"),
-
+
/**
* $CLASSPATH
*/
CLASSPATH("CLASSPATH"),
-
+
/**
* $APP_CLASSPATH
*/
APP_CLASSPATH("APP_CLASSPATH"),
-
+
/**
* $LD_LIBRARY_PATH
*/
LD_LIBRARY_PATH("LD_LIBRARY_PATH"),
-
+
/**
* $HADOOP_CONF_DIR
* Final, non-modifiable.
*/
HADOOP_CONF_DIR("HADOOP_CONF_DIR"),
-
+
/**
* $HADOOP_COMMON_HOME
*/
HADOOP_COMMON_HOME("HADOOP_COMMON_HOME"),
-
+
/**
* $HADOOP_HDFS_HOME
*/
HADOOP_HDFS_HOME("HADOOP_HDFS_HOME"),
-
+
/**
* $MALLOC_ARENA_MAX
*/
MALLOC_ARENA_MAX("MALLOC_ARENA_MAX"),
-
+
/**
* $HADOOP_YARN_HOME
*/
@@ -240,11 +234,11 @@ public interface ApplicationConstants {
private Environment(String variable) {
this.variable = variable;
}
-
+
public String key() {
return variable;
}
-
+
public String toString() {
return variable;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index f62ee50..42bf579 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -80,10 +80,10 @@ public class AMLauncher implements Runnable {
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
-
+
@SuppressWarnings("rawtypes")
private final EventHandler handler;
-
+
public AMLauncher(RMContext rmContext, RMAppAttempt application,
AMLauncherEventType eventType, Configuration conf) {
this.application = application;
@@ -93,20 +93,20 @@ public class AMLauncher implements Runnable {
this.handler = rmContext.getDispatcher().getEventHandler();
this.masterContainer = application.getMasterContainer();
}
-
+
private void connect() throws IOException {
ContainerId masterContainerID = masterContainer.getId();
-
+
containerMgrProxy = getContainerMgrProxy(masterContainerID);
}
-
+
private void launch() throws IOException, YarnException {
connect();
ContainerId masterContainerID = masterContainer.getId();
ApplicationSubmissionContext applicationContext =
application.getSubmissionContext();
LOG.info("Setting up container " + masterContainer
- + " for AM " + application.getAppAttemptId());
+ + " for AM " + application.getAppAttemptId());
ContainerLaunchContext launchContext =
createAMContainerLaunchContext(applicationContext, masterContainerID);
@@ -130,7 +130,7 @@ public class AMLauncher implements Runnable {
+ application.getAppAttemptId());
}
}
-
+
private void cleanup() throws IOException, YarnException {
connect();
ContainerId containerId = masterContainer.getId();
@@ -188,17 +188,17 @@ public class AMLauncher implements Runnable {
ContainerId containerID) throws IOException {
// Construct the actual Container
- ContainerLaunchContext container =
+ ContainerLaunchContext container =
applicationMasterContext.getAMContainerSpec();
LOG.info("Command to launch container "
+ containerID
+ " : "
+ StringUtils.arrayToString(container.getCommands().toArray(
new String[0])));
-
+
// Finalize the container
setupTokens(container, containerID);
-
+
return container;
}
@@ -210,7 +210,7 @@ public class AMLauncher implements Runnable {
Map<String, String> environment = container.getEnvironment();
environment.put(ApplicationConstants.APPLICATION_WEB_PROXY_BASE_ENV,
application.getWebProxyBase());
- // Set AppSubmitTime and MaxAppAttempts to be consumable by the AM.
+ // Set AppSubmitTime to be consumable by the AM.
ApplicationId applicationId =
application.getAppAttemptId().getApplicationId();
environment.put(
@@ -218,9 +218,6 @@ public class AMLauncher implements Runnable {
String.valueOf(rmContext.getRMApps()
.get(applicationId)
.getSubmitTime()));
- environment.put(ApplicationConstants.MAX_APP_ATTEMPTS_ENV,
- String.valueOf(rmContext.getRMApps().get(
- applicationId).getMaxAppAttempts()));
// Set flow context info
for (String tag :
rmContext.getRMApps().get(applicationId).getApplicationTags()) {
@@ -267,7 +264,7 @@ public class AMLauncher implements Runnable {
((RMAppAttemptImpl)application).setAMRMToken(amrmToken);
return amrmToken;
}
-
+
@SuppressWarnings("unchecked")
public void run() {
switch (eventType) {
@@ -297,7 +294,7 @@ public class AMLauncher implements Runnable {
sb.append(" is not handled by this NodeManager");
if (!e.getMessage().contains(sb.toString())) {
// Ignoring if container is already killed by Node Manager.
- LOG.info("Error cleaning master ", e);
+ LOG.info("Error cleaning master ", e);
}
}
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e06811e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
index 9a4395e..d4f8e93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
@@ -107,8 +107,7 @@ public class TestApplicationMasterLauncher {
nmHostAtContainerManager = tokenId.getNmHostAddress();
submitTimeAtContainerManager =
Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
- maxAppAttempts =
- Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
+ maxAppAttempts = YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS;
return StartContainersResponse.newInstance(
new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
new HashMap<ContainerId, SerializedException>());
@@ -184,8 +183,8 @@ public class TestApplicationMasterLauncher {
am.waitForState(RMAppAttemptState.FINISHED);
rm.stop();
}
-
-
+
+
@SuppressWarnings("unused")
@Test(timeout = 100000)
public void testallocateBeforeAMRegistration() throws Exception {
[34/50] [abbrv] hadoop git commit: YARN-3713. Remove duplicate
function call storeContainerDiagnostics in
ContainerDiagnosticsUpdateTransition (zxu via rkanter)
Posted by zj...@apache.org.
YARN-3713. Remove duplicate function call storeContainerDiagnostics in ContainerDiagnosticsUpdateTransition (zxu via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/704eee9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/704eee9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/704eee9e
Branch: refs/heads/YARN-2928
Commit: 704eee9e8b13968bbdbab2020ef445c508f5c239
Parents: 6b126e9
Author: Robert Kanter <rk...@apache.org>
Authored: Fri May 29 15:34:37 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:58 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../nodemanager/containermanager/container/ContainerImpl.java | 7 -------
2 files changed, 3 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/704eee9e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4ddf11b..906d9d4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -368,6 +368,9 @@ Release 2.8.0 - UNRELEASED
YARN-3722. Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils.
(Masatake Iwasaki via devaraj)
+ YARN-3713. Remove duplicate function call storeContainerDiagnostics in
+ ContainerDiagnosticsUpdateTransition (zxu via rkanter)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/704eee9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 9997ca2..289d6d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1105,13 +1105,6 @@ public class ContainerImpl implements Container {
ContainerDiagnosticsUpdateEvent updateEvent =
(ContainerDiagnosticsUpdateEvent) event;
container.addDiagnostics(updateEvent.getDiagnosticsUpdate(), "\n");
- try {
- container.stateStore.storeContainerDiagnostics(container.containerId,
- container.diagnostics);
- } catch (IOException e) {
- LOG.warn("Unable to update state store diagnostics for "
- + container.containerId, e);
- }
}
}
[42/50] [abbrv] hadoop git commit: HDFS-8490. Typo in trace enabled
log in ExceptionHandler of WebHDFS. Contributed by Archana T.
Posted by zj...@apache.org.
HDFS-8490. Typo in trace enabled log in ExceptionHandler of WebHDFS. Contributed by Archana T.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bc36be7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bc36be7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bc36be7
Branch: refs/heads/YARN-2928
Commit: 8bc36be7aa76b5f214854a49f14d2470719e606b
Parents: 00ccf73
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Mon Jun 1 00:41:50 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:13:00 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bc36be7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cc8235c..10e5da4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -825,6 +825,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7609. Avoid retry cache collision when Standby NameNode loading edits.
(Ming Ma via jing9)
+ HDFS-8490. Typo in trace enabled log in ExceptionHandler of WebHDFS.
+ (Archana T via ozawa)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bc36be7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
index 779a397..a6a8aa1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
@@ -50,7 +50,7 @@ class ExceptionHandler {
Exception e = cause instanceof Exception ? (Exception) cause : new Exception(cause);
if (LOG.isTraceEnabled()) {
- LOG.trace("GOT EXCEPITION", e);
+ LOG.trace("GOT EXCEPTION", e);
}
//Convert exception
[45/50] [abbrv] hadoop git commit: YARN-3725. App submission via REST
API is broken in secure mode due to Timeline DT service address is empty.
(Zhijie Shen via wangda)
Posted by zj...@apache.org.
YARN-3725. App submission via REST API is broken in secure mode due to Timeline DT service address is empty. (Zhijie Shen via wangda)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/732a70f1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/732a70f1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/732a70f1
Branch: refs/heads/YARN-2928
Commit: 732a70f12f5c9b84486503275bfe60a219aba44c
Parents: a977bed
Author: Wangda Tan <wa...@apache.org>
Authored: Sun May 31 16:30:34 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:04 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../client/api/impl/TimelineClientImpl.java | 30 ++++++++++++++------
.../TestTimelineAuthenticationFilter.java | 11 +++++++
3 files changed, 36 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/732a70f1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e770a5b..807165f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -697,6 +697,9 @@ Release 2.7.1 - UNRELEASED
YARN-2900. Application (Attempt and Container) Not Found in AHS results
in Internal Server Error (500). (Zhijie Shen and Mit Desai via xgong)
+ YARN-3725. App submission via REST API is broken in secure mode due to
+ Timeline DT service address is empty. (Zhijie Shen via wangda)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/732a70f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index e44a8a9..77e9af6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -522,9 +522,12 @@ public class TimelineClientImpl extends TimelineClient {
public long renewDelegationToken(
final Token<TimelineDelegationTokenIdentifier> timelineDT)
throws IOException, YarnException {
- boolean useHttps = YarnConfiguration.useHttps(this.getConfig());
- final String scheme = useHttps ? "https" : "http";
- final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT);
+ final boolean isTokenServiceAddrEmpty =
+ timelineDT.getService().toString().isEmpty();
+ final String scheme = isTokenServiceAddrEmpty ? null
+ : (YarnConfiguration.useHttps(this.getConfig()) ? "https" : "http");
+ final InetSocketAddress address = isTokenServiceAddrEmpty ? null
+ : SecurityUtil.getTokenServiceAddr(timelineDT);
PrivilegedExceptionAction<Long> renewDTAction =
new PrivilegedExceptionAction<Long>() {
@@ -539,7 +542,11 @@ public class TimelineClientImpl extends TimelineClient {
DelegationTokenAuthenticatedURL authUrl =
new DelegationTokenAuthenticatedURL(authenticator,
connConfigurator);
- final URI serviceURI = new URI(scheme, null, address.getHostName(),
+ // If the token service address is not available, fall back to use
+ // the configured service address.
+ final URI serviceURI = isTokenServiceAddrEmpty ?
+ constructResURI(getConfig(), getTimelineServiceAddress(), false)
+ : new URI(scheme, null, address.getHostName(),
address.getPort(), RESOURCE_URI_STR_V1, null, null);
return authUrl
.renewDelegationToken(serviceURI.toURL(), token, doAsUser);
@@ -553,9 +560,12 @@ public class TimelineClientImpl extends TimelineClient {
public void cancelDelegationToken(
final Token<TimelineDelegationTokenIdentifier> timelineDT)
throws IOException, YarnException {
- boolean useHttps = YarnConfiguration.useHttps(this.getConfig());
- final String scheme = useHttps ? "https" : "http";
- final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT);
+ final boolean isTokenServiceAddrEmpty =
+ timelineDT.getService().toString().isEmpty();
+ final String scheme = isTokenServiceAddrEmpty ? null
+ : (YarnConfiguration.useHttps(this.getConfig()) ? "https" : "http");
+ final InetSocketAddress address = isTokenServiceAddrEmpty ? null
+ : SecurityUtil.getTokenServiceAddr(timelineDT);
PrivilegedExceptionAction<Void> cancelDTAction =
new PrivilegedExceptionAction<Void>() {
@@ -570,7 +580,11 @@ public class TimelineClientImpl extends TimelineClient {
DelegationTokenAuthenticatedURL authUrl =
new DelegationTokenAuthenticatedURL(authenticator,
connConfigurator);
- final URI serviceURI = new URI(scheme, null, address.getHostName(),
+ // If the token service address is not available, fall back to use
+ // the configured service address.
+ final URI serviceURI = isTokenServiceAddrEmpty ?
+ constructResURI(getConfig(), getTimelineServiceAddress(), false)
+ : new URI(scheme, null, address.getHostName(),
address.getPort(), RESOURCE_URI_STR_V1, null, null);
authUrl.cancelDelegationToken(serviceURI.toURL(), token, doAsUser);
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/732a70f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
index c93e8f2..063f512 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
@@ -240,12 +240,21 @@ public class TestTimelineAuthenticationFilter {
Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner());
// Renew token
+ Assert.assertFalse(token.getService().toString().isEmpty());
+ // Renew the token from the token service address
long renewTime1 = httpUserClient.renewDelegationToken(token);
Thread.sleep(100);
+ token.setService(new Text());
+ Assert.assertTrue(token.getService().toString().isEmpty());
+ // If the token service address is not avaiable, it still can be renewed
+ // from the configured address
long renewTime2 = httpUserClient.renewDelegationToken(token);
Assert.assertTrue(renewTime1 < renewTime2);
// Cancel token
+ Assert.assertTrue(token.getService().toString().isEmpty());
+ // If the token service address is not avaiable, it still can be canceled
+ // from the configured address
httpUserClient.cancelDelegationToken(token);
// Renew should not be successful because the token is canceled
try {
@@ -280,6 +289,8 @@ public class TestTimelineAuthenticationFilter {
Assert.assertTrue(renewTime1 < renewTime2);
// Cancel token
+ Assert.assertFalse(tokenToRenew.getService().toString().isEmpty());
+ // Cancel the token from the token service address
fooUserClient.cancelDelegationToken(tokenToRenew);
// Renew should not be successful because the token is canceled
[13/50] [abbrv] hadoop git commit: HADOOP-12030. test-patch should
only report on newly introduced findbugs warnings. (Sean Busbey via aw)
Posted by zj...@apache.org.
HADOOP-12030. test-patch should only report on newly introduced findbugs warnings. (Sean Busbey via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac3000a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac3000a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac3000a2
Branch: refs/heads/YARN-2928
Commit: ac3000a283e10e69635b25b1e25aabc779602ba3
Parents: 3eb698c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 09:53:50 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 258 +++++++++++++++----
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
2 files changed, 211 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac3000a2/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 5276023..f760951 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -44,6 +44,7 @@ function setup_defaults
LOAD_SYSTEM_PLUGINS=true
FINDBUGS_HOME=${FINDBUGS_HOME:-}
+ FINDBUGS_WARNINGS_FAIL_PRECHECK=false
ECLIPSE_HOME=${ECLIPSE_HOME:-}
BUILD_NATIVE=${BUILD_NATIVE:-true}
PATCH_BRANCH=""
@@ -585,6 +586,7 @@ function hadoop_usage
echo "--debug If set, then output some extra stuff to stderr"
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
+ echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
echo "--modulelist=<list> Specify additional modules to test (comma delimited)"
echo "--offline Avoid connecting to the Internet"
@@ -666,6 +668,9 @@ function parse_args
--findbugs-home=*)
FINDBUGS_HOME=${i#*=}
;;
+ --findbugs-strict-precheck)
+ FINDBUGS_WARNINGS_FAIL_PRECHECK=true
+ ;;
--git-cmd=*)
GIT=${i#*=}
;;
@@ -1060,6 +1065,12 @@ function precheck_without_patch
echo "Patch does not appear to need site tests."
fi
+ precheck_findbugs
+
+ if [[ $? != 0 ]] ; then
+ return 1
+ fi
+
add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
return 0
}
@@ -1838,40 +1849,84 @@ function check_mvn_install
return ${retval}
}
-## @description Verify patch does not trigger any findbugs warnings
+## @description are the needed bits for findbugs present?
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 findbugs will work for our use
+## @return 1 findbugs is missing some component
+function findbugs_is_installed
+{
+ if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
+ printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
+ add_jira_table -1 findbugs "Findbugs is not installed."
+ return 1
+ fi
+ return 0
+}
+
+## @description Run the maven findbugs plugin and record found issues in a bug database
## @audience private
## @stability evolving
## @replaceable no
## @return 0 on success
## @return 1 on failure
-function check_findbugs
+function findbugs_mvnrunner
{
- local findbugs_version
- local modules=${CHANGED_MODULES}
- local rc=0
- local module_suffix
- local findbugsWarnings=0
- local relative_file
- local newFindbugsWarnings
- local findbugsWarnings
- local line
- local firstpart
- local secondpart
+ local name=$1
+ local logfile=$2
+ local warnings_file=$3
- big_console_header "Determining number of patched Findbugs warnings."
+ echo_and_redirect "${logfile}" "${MVN}" clean test findbugs:findbugs -DskipTests \
+ "-D${PROJECT_NAME}PatchProcess" < /dev/null
+ if [[ $? != 0 ]]; then
+ return 1
+ fi
+ cp target/findbugsXml.xml "${warnings_file}.xml"
+
+ "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \
+ "${warnings_file}.xml" "${warnings_file}.xml"
+ if [[ $? != 0 ]]; then
+ return 1
+ fi
+ "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \
+ "${warnings_file}.html"
+ if [[ $? != 0 ]]; then
+ return 1
+ fi
+
+ return 0
+}
+
+## @description Track pre-existing findbugs warnings
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function precheck_findbugs
+{
+ local -r mypwd=$(pwd)
+ local module_suffix
+ local modules=${CHANGED_MODULES}
+ local module
+ local findbugs_version
+ local rc=0
+ local module_findbugs_warnings
+ local findbugs_warnings=0
verify_needed_test findbugs
+
if [[ $? == 0 ]]; then
- echo "Patch does not touch any java files. Skipping findbugs."
+ echo "Patch does not appear to need findbugs tests."
return 0
fi
- start_clock
+ echo "findbugs baseline for ${mypwd}"
- if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
- printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
- add_jira_table -1 findbugs "Findbugs is not installed."
+ findbugs_is_installed
+ if [[ $? != 0 ]]; then
return 1
fi
@@ -1880,67 +1935,170 @@ function check_findbugs
pushd "${module}" >/dev/null
echo " Running findbugs in ${module}"
module_suffix=$(basename "${module}")
- echo_and_redirect "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" "${MVN}" clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess \
- < /dev/null
+ findbugs_mvnrunner "${PATCH_BRANCH}" \
+ "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \
+ "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}"
(( rc = rc + $? ))
+
+ if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
+ #shellcheck disable=SC2016
+ module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
+ "${PATCH_BRANCH}" \
+ "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
+ "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
+ | ${AWK} '{print $1}')
+ if [[ $? != 0 ]]; then
+ popd >/dev/null
+ return 1
+ fi
+
+ findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings))
+
+ if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
+ add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html"
+ fi
+ fi
popd >/dev/null
done
#shellcheck disable=SC2016
- findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
+ findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt")
if [[ ${rc} -ne 0 ]]; then
- add_jira_table -1 findbugs "The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
+ echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?"
+ add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}."
+ return 1
+ fi
+
+ if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \
+ ${findbugs_warnings} -gt 0 ]] ; then
+ echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings."
+ add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings."
+ return 1
+ fi
+
+ return 0
+}
+
+## @description Verify patch does not trigger any findbugs warnings
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function check_findbugs
+{
+ local rc=0
+ local module
+ local modules=${CHANGED_MODULES}
+ local module_suffix
+ local combined_xml
+ local newBugs
+ local new_findbugs_warnings
+ local new_findbugs_fixed_warnings
+ local findbugs_warnings=0
+ local findbugs_fixed_warnings=0
+ local line
+ local firstpart
+ local secondpart
+ local findbugs_version
+
+ verify_needed_test findbugs
+
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "Determining number of patched Findbugs warnings."
+
+ start_clock
+
+ findbugs_is_installed
+ if [[ $? != 0 ]]; then
return 1
fi
- while read file
+ for module in ${modules}
do
- relative_file=${file#${BASEDIR}/} # strip leading ${BASEDIR} prefix
- if [[ ${relative_file} != "target/findbugsXml.xml" ]]; then
- module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
- module_suffix=$(basename "${module_suffix}")
- fi
+ pushd "${module}" >/dev/null
+ echo " Running findbugs in ${module}"
+ module_suffix=$(basename "${module}")
- cp "${file}" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
+ findbugs_mvnrunner patch \
+ "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \
+ "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}"
- "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -timestamp "01/01/2000" \
- "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \
- "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
+ if [[ $? != 0 ]] ; then
+ ((rc = rc +1))
+ echo "Post-patch findbugs compilation is broken."
+ add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken."
+ continue
+ fi
- #shellcheck disable=SC2016
- newFindbugsWarnings=$("${FINDBUGS_HOME}/bin/filterBugs" \
- -first "01/01/2000" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \
- "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \
- | ${AWK} '{print $1}')
+ combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml"
+ newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}"
+ "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
+ -output "${combined_xml}" \
+ "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \
+ "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
+ if [[ $? != 0 ]]; then
+ popd >/dev/null
+ return 1
+ fi
- echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
+ #shellcheck disable=SC2016
+ new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
+ "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
+ if [[ $? != 0 ]]; then
+ popd >/dev/null
+ return 1
+ fi
+ #shellcheck disable=SC2016
+ new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
+ "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
+ if [[ $? != 0 ]]; then
+ popd >/dev/null
+ return 1
+ fi
- findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
+ echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
+ findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
+ findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
- "${FINDBUGS_HOME}/bin/convertXmlToText" -html \
- "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \
- "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.html"
+ "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \
+ "${newBugs}.html"
+ if [[ $? != 0 ]]; then
+ popd >/dev/null
+ return 1
+ fi
- if [[ ${newFindbugsWarnings} -gt 0 ]] ; then
+ if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
populate_test_table FindBugs "module:${module_suffix}"
while read line; do
firstpart=$(echo "${line}" | cut -f2 -d:)
secondpart=$(echo "${line}" | cut -f9- -d' ')
add_jira_test_table "" "${firstpart}:${secondpart}"
- done < <("${FINDBUGS_HOME}/bin/convertXmlToText" \
- "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml")
+ done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml")
add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
fi
- done < <(find "${BASEDIR}" -name findbugsXml.xml)
- if [[ ${findbugsWarnings} -gt 0 ]] ; then
- add_jira_table -1 findbugs "The patch appears to introduce ${findbugsWarnings} new Findbugs (version ${findbugs_version}) warnings."
+ popd >/dev/null
+ done
+
+ #shellcheck disable=SC2016
+ findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
+
+ if [[ ${findbugs_warnings} -gt 0 ]] ; then
+ add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings."
return 1
fi
- add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
+ if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then
+ add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings."
+ else
+ add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
+ fi
return 0
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac3000a2/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1009d1a..eb1db29 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -606,6 +606,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11594. Improve the readability of site index of documentation.
(Masatake Iwasaki via aajisaka)
+ HADOOP-12030. test-patch should only report on newly introduced
+ findbugs warnings. (Sean Busbey via aw)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
[15/50] [abbrv] hadoop git commit: HADOOP-12004. test-patch breaks
with reexec in certain situations (Sean Busbey via aw)
Posted by zj...@apache.org.
HADOOP-12004. test-patch breaks with reexec in certain situations (Sean Busbey via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dc7948e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dc7948e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dc7948e
Branch: refs/heads/YARN-2928
Commit: 7dc7948e3ca41ee4b54f85f55d7235ee7ed5e85b
Parents: 822cac2
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 08:15:17 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 2 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++++
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dc7948e/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index d5d6c26..5276023 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1509,7 +1509,7 @@ function check_reexec
exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \
--reexec \
- --branch "${PATCH_BRANCH}" \
+ --branch="${PATCH_BRANCH}" \
--patch-dir="${PATCH_DIR}" \
"${USER_PARAMS[@]}"
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dc7948e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d29a885..48e9f7d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -624,6 +624,7 @@ Release 2.8.0 - UNRELEASED
IPC server. (Masatake Iwasaki via aajisaka)
BUG FIXES
+
HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there
is an I/O error during requestShortCircuitShm (cmccabe)
@@ -761,6 +762,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-8751. NPE in Token.toString() when Token is constructed using null
identifier. (kanaka kumar avvaru via aajisaka)
+ HADOOP-12004. test-patch breaks with reexec in certain situations (Sean
+ Busbey via aw)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
[31/50] [abbrv] hadoop git commit: HADOOP-12042. Users may see
TrashPolicy if hdfs dfs -rm is run (Contributed by Andreina J)
Posted by zj...@apache.org.
HADOOP-12042. Users may see TrashPolicy if hdfs dfs -rm is run (Contributed by Andreina J)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab02e318
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab02e318
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab02e318
Branch: refs/heads/YARN-2928
Commit: ab02e3187cbabe9b20a730f0d4f7f775f8c1134f
Parents: 0f26da6
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri May 29 16:51:23 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/fs/TrashPolicyDefault.java | 7 ++++---
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab02e318/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index af82c0b..bc74696 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -797,6 +797,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11959. WASB should configure client side socket timeout in storage
client blob request options. (Ivan Mitic via cnauroth)
+ HADOOP-12042. Users may see TrashPolicy if hdfs dfs -rm is run
+ (Andreina J via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab02e318/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index d6a9b4b..88aeab5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -89,9 +89,6 @@ public class TrashPolicyDefault extends TrashPolicy {
this.emptierInterval = (long)(conf.getFloat(
FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT)
* MSECS_PER_MINUTE);
- LOG.info("Namenode trash configuration: Deletion interval = " +
- (this.deletionInterval / MSECS_PER_MINUTE) + " minutes, Emptier interval = " +
- (this.emptierInterval / MSECS_PER_MINUTE) + " minutes.");
}
private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
@@ -251,6 +248,10 @@ public class TrashPolicyDefault extends TrashPolicy {
" minutes that is used for deletion instead");
this.emptierInterval = deletionInterval;
}
+ LOG.info("Namenode trash configuration: Deletion interval = "
+ + (deletionInterval / MSECS_PER_MINUTE)
+ + " minutes, Emptier interval = "
+ + (emptierInterval / MSECS_PER_MINUTE) + " minutes.");
}
@Override
[37/50] [abbrv] hadoop git commit: HADOOP-12043. Display warning if
defaultFs is not set when running fs commands. Contributed by Eddy Xu.
Posted by zj...@apache.org.
HADOOP-12043. Display warning if defaultFs is not set when running fs commands. Contributed by Eddy Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/312030a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/312030a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/312030a3
Branch: refs/heads/YARN-2928
Commit: 312030a3122d10b829c16ab655225128e5434ecb
Parents: 704eee9
Author: Andrew Wang <wa...@apache.org>
Authored: Fri May 29 17:15:58 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:58 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../fs/CommonConfigurationKeysPublic.java | 6 ++++
.../org/apache/hadoop/fs/shell/FsCommand.java | 30 +++++++++++++++++++-
.../java/org/apache/hadoop/fs/shell/Ls.java | 8 ++++++
.../src/main/resources/core-default.xml | 8 ++++++
.../java/org/apache/hadoop/fs/shell/TestLs.java | 25 ++++++++++++++++
6 files changed, 79 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e5d2e39..00075b4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating
(Masatake Iwasaki via Colin P. McCabe)
+ HADOOP-12043. Display warning if defaultFs is not set when running fs
+ commands. (Lei Xu via wang)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 90c6934..7231d59 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -363,5 +363,11 @@ public class CommonConfigurationKeysPublic {
"hadoop.security.random.device.file.path";
public static final String HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT =
"/dev/urandom";
+
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY =
+ "hadoop.shell.missing.defaultFs.warning";
+ public static final boolean HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT =
+ false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
index 9515fde..971950f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.shell;
import java.io.IOException;
+import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -27,8 +28,13 @@ import org.apache.hadoop.fs.FsShellPermissions;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.find.Find;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
+
/**
- * Base class for all "hadoop fs" commands
+ * Base class for all "hadoop fs" commands.
*/
@InterfaceAudience.Private
@@ -90,4 +96,26 @@ abstract public class FsCommand extends Command {
public int runAll() {
return run(args);
}
+
+ @Override
+ protected void processRawArguments(LinkedList<String> args)
+ throws IOException {
+ LinkedList<PathData> expendedArgs = expandArguments(args);
+ // If "fs.defaultFs" is not set appropriately, it warns the user that the
+ // command is not running against HDFS.
+ final boolean displayWarnings = getConf().getBoolean(
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY,
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT);
+ if (displayWarnings) {
+ final String defaultFs = getConf().get(FS_DEFAULT_NAME_KEY);
+ final boolean missingDefaultFs =
+ defaultFs == null || defaultFs.equals(FS_DEFAULT_NAME_DEFAULT);
+ if (missingDefaultFs) {
+ err.printf(
+ "Warning: fs.defaultFs is not set when running \"%s\" command.%n",
+ getCommandName());
+ }
+ }
+ processArguments(expendedArgs);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index d5c52ed..67348c6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.LinkedList;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -102,6 +104,12 @@ class Ls extends FsCommand {
protected boolean humanReadable = false;
+ protected Ls() {}
+
+ protected Ls(Configuration conf) {
+ super(conf);
+ }
+
protected String formatSize(long size) {
return humanReadable
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index a1bc780..ae23a75 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1899,4 +1899,12 @@ for ldap providers in the same way as above does.
<value>Client</value>
</property>
+ <property>
+ <description>
+ Enable hdfs shell commands to display warnings if (fs.defaultFS) property
+ is not set.
+ </description>
+ <name>hadoop.shell.missing.defaultFs.warning</name>
+ <value>false</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/312030a3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
index 4a9103f..cba44ee 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
@@ -17,10 +17,12 @@
*/
package org.apache.hadoop.fs.shell;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
import static org.junit.Assert.*;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.*;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
@@ -1048,6 +1050,29 @@ public class TestLs {
verifyNoMoreInteractions(out);
}
+ private static void displayWarningOnLocalFileSystem(boolean shouldDisplay)
+ throws IOException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, shouldDisplay);
+
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+ PrintStream err = new PrintStream(buf, true);
+ Ls ls = new Ls(conf);
+ ls.err = err;
+ ls.run("file:///.");
+ assertEquals(shouldDisplay, buf.toString().contains(
+ "Warning: fs.defaultFs is not set when running \"ls\" command."));
+ }
+
+ @Test
+ public void displayWarningsOnLocalFileSystem() throws IOException {
+ // Display warnings.
+ displayWarningOnLocalFileSystem(true);
+ // Does not display warnings.
+ displayWarningOnLocalFileSystem(false);
+ }
+
// check the deprecated flag isn't set
@Test
public void isDeprecated() {
[16/50] [abbrv] hadoop git commit: HDFS-8407. libhdfs
hdfsListDirectory must set errno to 0 on success (Masatake Iwasaki via Colin
P. McCabe)
Posted by zj...@apache.org.
HDFS-8407. libhdfs hdfsListDirectory must set errno to 0 on success (Masatake Iwasaki via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61510b9d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61510b9d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61510b9d
Branch: refs/heads/YARN-2928
Commit: 61510b9deb0365e250736ccddbe8b4abddc869fa
Parents: b914cca
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu May 28 11:23:44 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/main/native/libhdfs/expect.h | 18 ++++++++++++++++++
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 1 +
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 3 ++-
.../main/native/libhdfs/test/test_libhdfs_ops.c | 9 +++++++++
.../main/native/libhdfs/test_libhdfs_threaded.c | 17 ++++++++++++++++-
6 files changed, 49 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 47481a7..1c9b25e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -813,6 +813,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8431. hdfs crypto class not found in Windows.
(Anu Engineer via cnauroth)
+ HDFS-8407. hdfsListDirectory must set errno to 0 on success (Masatake
+ Iwasaki via Colin P. McCabe)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h
index e64b108..49aa285 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/expect.h
@@ -48,6 +48,24 @@ struct hdfsFile_internal;
} \
} while (0);
+#define EXPECT_NULL_WITH_ERRNO(x, e) \
+ do { \
+ const void* __my_ret__ = x; \
+ int __my_errno__ = errno; \
+ if (__my_ret__ != NULL) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \
+ "got non-NULL value %p from %s\n", \
+ __FILE__, __LINE__, __my_errno__, __my_ret__, #x); \
+ return -1; \
+ } \
+ if (__my_errno__ != e) { \
+ fprintf(stderr, "TEST_ERROR: failed on %s:%d (errno: %d): " \
+ "got expected NULL without expected errno %d from %s\n", \
+ __FILE__, __LINE__, __my_errno__, e, #x); \
+ return -1; \
+ } \
+ } while (0);
+
#define EXPECT_NONNULL(x) \
do { \
const void* __my_ret__ = x; \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
index 504d47e..a3769fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c
@@ -3257,6 +3257,7 @@ done:
return NULL;
}
*numEntries = jPathListSize;
+ errno = 0;
return pathList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
index 5b7bc1e..c1515d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h
@@ -671,7 +671,8 @@ extern "C" {
* @param path The path of the directory.
* @param numEntries Set to the number of files/directories in path.
* @return Returns a dynamically-allocated array of hdfsFileInfo
- * objects; NULL on error.
+ * objects; NULL on error or empty directory.
+ * errno is set to non-zero on error or zero on success.
*/
LIBHDFS_EXTERNAL
hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c
index aa9441a..f564de4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_ops.c
@@ -307,6 +307,15 @@ int main(int argc, char **argv) {
}
fileList = 0;
+ fileList = hdfsListDirectory(fs, newDirectory, &numEntries);
+ if (!(fileList == NULL && numEntries == 0 && !errno)) {
+ fprintf(stderr, "waah! hdfsListDirectory for empty %s - FAILED!\n", newDirectory);
+ totalResult++;
+ } else {
+ fprintf(stderr, "hdfsListDirectory for empty %s - SUCCESS!\n", newDirectory);
+ }
+
+ fileList = 0;
if((fileList = hdfsListDirectory(fs, slashTmp, &numEntries)) != NULL) {
for(i=0; i < numEntries; ++i) {
fprintf(stderr, "Name: %s, ", fileList[i].mName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61510b9d/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
index 17ff7a8..702430c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c
@@ -142,7 +142,7 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
{
char tmp[4096];
hdfsFile file;
- int ret, expected;
+ int ret, expected, numEntries;
hdfsFileInfo *fileInfo;
struct hdfsReadStatistics *readStats = NULL;
@@ -153,6 +153,14 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
EXPECT_ZERO(doTestGetDefaultBlockSize(fs, paths->prefix));
+ /* There should be no entry in the directory. */
+ errno = EACCES; // see if errno is set to 0 on success
+ EXPECT_NULL_WITH_ERRNO(hdfsListDirectory(fs, paths->prefix, &numEntries), 0);
+ if (numEntries != 0) {
+ fprintf(stderr, "hdfsListDirectory set numEntries to "
+ "%d on empty directory.", numEntries);
+ }
+
/* There should not be any file to open for reading. */
EXPECT_NULL(hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0));
@@ -179,6 +187,13 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs,
EXPECT_ZERO(hdfsHSync(fs, file));
EXPECT_ZERO(hdfsCloseFile(fs, file));
+ /* There should be 1 entry in the directory. */
+ EXPECT_NONNULL(hdfsListDirectory(fs, paths->prefix, &numEntries));
+ if (numEntries != 1) {
+ fprintf(stderr, "hdfsListDirectory set numEntries to "
+ "%d on directory containing 1 file.", numEntries);
+ }
+
/* Let's re-open the file for reading */
file = hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0);
EXPECT_NONNULL(file);
[21/50] [abbrv] hadoop git commit: HADOOP-11142. Remove hdfs dfs
reference from file system shell documentation (Kengo Seki via aw)
Posted by zj...@apache.org.
HADOOP-11142. Remove hdfs dfs reference from file system shell documentation (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82829c82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82829c82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82829c82
Branch: refs/heads/YARN-2928
Commit: 82829c8293eea24806eb2e44112e5abcd80add29
Parents: 6c65e8a
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 10:57:04 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/site/markdown/FileSystemShell.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82829c82/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7da02ed..8de5ae4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -221,6 +221,9 @@ Trunk (Unreleased)
HADOOP-12016. Typo in FileSystem::listStatusIterator
(Arthur Vigil via jghoman)
+ HADOOP-11142. Remove hdfs dfs reference from file system shell
+ documentation (Kengo Seki via aw)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82829c82/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 8ecc24a..144cb73 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -185,7 +185,7 @@ Example:
* `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
* `hadoop fs -count -q hdfs://nn1.example.com/file1`
* `hadoop fs -count -q -h hdfs://nn1.example.com/file1`
-* `hdfs dfs -count -q -h -v hdfs://nn1.example.com/file1`
+* `hadoop fs -count -q -h -v hdfs://nn1.example.com/file1`
Exit Code:
[48/50] [abbrv] hadoop git commit: HDFS-8489. Subclass BlockInfo to
represent contiguous blocks. Contributed by Zhe Zhang.
Posted by zj...@apache.org.
HDFS-8489. Subclass BlockInfo to represent contiguous blocks. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/065ab4d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/065ab4d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/065ab4d7
Branch: refs/heads/YARN-2928
Commit: 065ab4d7528df8a099986b6009be7d864d8695bd
Parents: 7505e35
Author: Jing Zhao <ji...@apache.org>
Authored: Mon Jun 1 11:37:47 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:22 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/blockmanagement/BlockInfo.java | 74 +++--------
.../blockmanagement/BlockInfoContiguous.java | 123 +++++++++++++++++++
.../BlockInfoContiguousUnderConstruction.java | 7 +-
.../server/blockmanagement/BlockManager.java | 5 +-
.../hdfs/server/blockmanagement/BlocksMap.java | 13 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../hdfs/server/namenode/FSImageFormat.java | 3 +-
.../server/namenode/FSImageFormatPBINode.java | 4 +-
.../server/namenode/FSImageSerialization.java | 3 +-
.../snapshot/FSImageFormatPBSnapshot.java | 3 +-
.../server/blockmanagement/TestBlockInfo.java | 14 +--
.../blockmanagement/TestBlockManager.java | 4 +-
.../blockmanagement/TestDatanodeDescriptor.java | 4 +-
.../blockmanagement/TestPendingReplication.java | 2 +-
.../blockmanagement/TestReplicationPolicy.java | 2 +-
.../hdfs/server/namenode/CreateEditsLog.java | 5 +-
.../TestCommitBlockSynchronization.java | 5 +-
.../hdfs/server/namenode/TestINodeFile.java | 3 +-
.../namenode/TestTruncateQuotaUpdate.java | 3 +-
.../snapshot/TestFileWithSnapshotFeature.java | 3 +-
21 files changed, 186 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 10e5da4..1f7a36a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -588,6 +588,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8443. Document dfs.namenode.service.handler.count in hdfs-site.xml.
(J.Andreina via aajisaka)
+ HDFS-8489. Subclass BlockInfo to represent contiguous blocks.
+ (Zhe Zhang via jing9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
index 0370c57..dea31c4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.util.LightWeightGSet;
* the block are stored.
*/
@InterfaceAudience.Private
-public class BlockInfo extends Block
+public abstract class BlockInfo extends Block
implements LightWeightGSet.LinkedElement {
public static final BlockInfo[] EMPTY_ARRAY = {};
@@ -51,7 +51,7 @@ public class BlockInfo extends Block
* per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
* bytes using the triplets.
*/
- private Object[] triplets;
+ protected Object[] triplets;
/**
* Construct an entry for blocksmap
@@ -102,7 +102,7 @@ public class BlockInfo extends Block
return (DatanodeStorageInfo)triplets[index*3];
}
- private BlockInfo getPrevious(int index) {
+ BlockInfo getPrevious(int index) {
assert this.triplets != null : "BlockInfo is not initialized";
assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
BlockInfo info = (BlockInfo)triplets[index*3+1];
@@ -122,7 +122,7 @@ public class BlockInfo extends Block
return info;
}
- private void setStorageInfo(int index, DatanodeStorageInfo storage) {
+ void setStorageInfo(int index, DatanodeStorageInfo storage) {
assert this.triplets != null : "BlockInfo is not initialized";
assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
triplets[index*3] = storage;
@@ -136,7 +136,7 @@ public class BlockInfo extends Block
* @param to - block to be set to previous on the list of blocks
* @return current previous block on the list of blocks
*/
- private BlockInfo setPrevious(int index, BlockInfo to) {
+ BlockInfo setPrevious(int index, BlockInfo to) {
assert this.triplets != null : "BlockInfo is not initialized";
assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
BlockInfo info = (BlockInfo)triplets[index*3+1];
@@ -152,7 +152,7 @@ public class BlockInfo extends Block
* @param to - block to be set to next on the list of blocks
* * @return current next block on the list of blocks
*/
- private BlockInfo setNext(int index, BlockInfo to) {
+ BlockInfo setNext(int index, BlockInfo to) {
assert this.triplets != null : "BlockInfo is not initialized";
assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
BlockInfo info = (BlockInfo)triplets[index*3+2];
@@ -167,68 +167,26 @@ public class BlockInfo extends Block
}
/**
- * Ensure that there is enough space to include num more triplets.
- * @return first free triplet index.
+ * Count the number of data-nodes the block belongs to.
*/
- private int ensureCapacity(int num) {
- assert this.triplets != null : "BlockInfo is not initialized";
- int last = numNodes();
- if(triplets.length >= (last+num)*3)
- return last;
- /* Not enough space left. Create a new array. Should normally
- * happen only when replication is manually increased by the user. */
- Object[] old = triplets;
- triplets = new Object[(last+num)*3];
- System.arraycopy(old, 0, triplets, 0, last*3);
- return last;
- }
+ public abstract int numNodes();
/**
- * Count the number of data-nodes the block belongs to.
+ * Add a {@link DatanodeStorageInfo} location for a block.
*/
- public int numNodes() {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert triplets.length % 3 == 0 : "Malformed BlockInfo";
- for(int idx = getCapacity()-1; idx >= 0; idx--) {
- if(getDatanode(idx) != null)
- return idx+1;
- }
- return 0;
- }
+ abstract boolean addStorage(DatanodeStorageInfo storage);
/**
- * Add a {@link DatanodeStorageInfo} location for a block
+ * Remove {@link DatanodeStorageInfo} location for a block
*/
- boolean addStorage(DatanodeStorageInfo storage) {
- // find the last null node
- int lastNode = ensureCapacity(1);
- setStorageInfo(lastNode, storage);
- setNext(lastNode, null);
- setPrevious(lastNode, null);
- return true;
- }
+ abstract boolean removeStorage(DatanodeStorageInfo storage);
+
/**
- * Remove {@link DatanodeStorageInfo} location for a block
+ * Replace the current BlockInfo with the new one in corresponding
+ * DatanodeStorageInfo's linked list
*/
- boolean removeStorage(DatanodeStorageInfo storage) {
- int dnIndex = findStorageInfo(storage);
- if(dnIndex < 0) // the node is not found
- return false;
- assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
- "Block is still in the list and must be removed first.";
- // find the last not null node
- int lastNode = numNodes()-1;
- // replace current node triplet by the lastNode one
- setStorageInfo(dnIndex, getStorageInfo(lastNode));
- setNext(dnIndex, getNext(lastNode));
- setPrevious(dnIndex, getPrevious(lastNode));
- // set the last triplet to null
- setStorageInfo(lastNode, null);
- setNext(lastNode, null);
- setPrevious(lastNode, null);
- return true;
- }
+ abstract void replaceBlock(BlockInfo newBlock);
/**
* Find specified DatanodeStorageInfo.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
new file mode 100644
index 0000000..eff89a8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+
+/**
+ * Subclass of {@link BlockInfo}, used for a block with replication scheme.
+ */
+@InterfaceAudience.Private
+public class BlockInfoContiguous extends BlockInfo {
+ public static final BlockInfoContiguous[] EMPTY_ARRAY = {};
+
+ public BlockInfoContiguous(short size) {
+ super(size);
+ }
+
+ public BlockInfoContiguous(Block blk, short size) {
+ super(blk, size);
+ }
+
+ /**
+ * Copy construction.
+ * This is used to convert BlockReplicationInfoUnderConstruction
+ * @param from BlockReplicationInfo to copy from.
+ */
+ protected BlockInfoContiguous(BlockInfoContiguous from) {
+ super(from);
+ }
+
+ /**
+ * Ensure that there is enough space to include num more triplets.
+ * @return first free triplet index.
+ */
+ private int ensureCapacity(int num) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ int last = numNodes();
+ if (triplets.length >= (last+num)*3) {
+ return last;
+ }
+ /* Not enough space left. Create a new array. Should normally
+ * happen only when replication is manually increased by the user. */
+ Object[] old = triplets;
+ triplets = new Object[(last+num)*3];
+ System.arraycopy(old, 0, triplets, 0, last * 3);
+ return last;
+ }
+
+ @Override
+ boolean addStorage(DatanodeStorageInfo storage) {
+ // find the last null node
+ int lastNode = ensureCapacity(1);
+ setStorageInfo(lastNode, storage);
+ setNext(lastNode, null);
+ setPrevious(lastNode, null);
+ return true;
+ }
+
+ @Override
+ boolean removeStorage(DatanodeStorageInfo storage) {
+ int dnIndex = findStorageInfo(storage);
+ if (dnIndex < 0) { // the node is not found
+ return false;
+ }
+ assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+ "Block is still in the list and must be removed first.";
+ // find the last not null node
+ int lastNode = numNodes()-1;
+ // replace current node triplet by the lastNode one
+ setStorageInfo(dnIndex, getStorageInfo(lastNode));
+ setNext(dnIndex, getNext(lastNode));
+ setPrevious(dnIndex, getPrevious(lastNode));
+ // set the last triplet to null
+ setStorageInfo(lastNode, null);
+ setNext(lastNode, null);
+ setPrevious(lastNode, null);
+ return true;
+ }
+
+ @Override
+ public int numNodes() {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+
+ for (int idx = getCapacity()-1; idx >= 0; idx--) {
+ if (getDatanode(idx) != null) {
+ return idx + 1;
+ }
+ }
+ return 0;
+ }
+
+ @Override
+ void replaceBlock(BlockInfo newBlock) {
+ assert newBlock instanceof BlockInfoContiguous;
+ for (int i = this.numNodes() - 1; i >= 0; i--) {
+ final DatanodeStorageInfo storage = this.getStorageInfo(i);
+ final boolean removed = storage.removeBlock(this);
+ assert removed : "currentBlock not found.";
+
+ final DatanodeStorageInfo.AddBlockResult result = storage.addBlock(
+ newBlock);
+ assert result == DatanodeStorageInfo.AddBlockResult.ADDED :
+ "newBlock already exists.";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
index 58020a6..b757171 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
* Represents a block that is currently being constructed.<br>
* This is usually the last block of a file opened for write or append.
*/
-public class BlockInfoContiguousUnderConstruction extends BlockInfo {
+public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
/** Block state. See {@link BlockUCState} */
private BlockUCState blockUCState;
@@ -165,7 +165,8 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfo {
/**
* Create a block that is currently being constructed.
*/
- public BlockInfoContiguousUnderConstruction(Block blk, short replication, BlockUCState state, DatanodeStorageInfo[] targets) {
+ public BlockInfoContiguousUnderConstruction(Block blk, short replication,
+ BlockUCState state, DatanodeStorageInfo[] targets) {
super(blk, replication);
assert getBlockUCState() != BlockUCState.COMPLETE :
"BlockInfoUnderConstruction cannot be in COMPLETE state";
@@ -185,7 +186,7 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfo {
BlockInfo convertToCompleteBlock() throws IOException {
assert getBlockUCState() != BlockUCState.COMPLETE :
"Trying to convert a COMPLETE block";
- return new BlockInfo(this);
+ return new BlockInfoContiguous(this);
}
/** Set expected locations */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 45a485b..d18d7fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1765,7 +1765,8 @@ public class BlockManager {
BlockToMarkCorrupt(BlockInfo stored, long gs, String reason,
Reason reasonCode) {
- this(new BlockInfo(stored), stored, reason, reasonCode);
+ this(new BlockInfoContiguous((BlockInfoContiguous)stored), stored,
+ reason, reasonCode);
//the corrupted block in datanode has a different generation stamp
corrupted.setGenerationStamp(gs);
}
@@ -2144,7 +2145,7 @@ public class BlockManager {
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
- BlockInfo delimiter = new BlockInfo(new Block(), (short) 1);
+ BlockInfo delimiter = new BlockInfoContiguous(new Block(), (short) 1);
AddBlockResult result = storageInfo.addBlock(delimiter);
assert result == AddBlockResult.ADDED
: "Delimiting block cannot be present in the node";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index d06d503..9eb1059 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -20,12 +20,10 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.Iterator;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
-import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
@@ -222,16 +220,7 @@ class BlocksMap {
BlockInfo currentBlock = blocks.get(newBlock);
assert currentBlock != null : "the block if not in blocksMap";
// replace block in data-node lists
- for (int i = currentBlock.numNodes() - 1; i >= 0; i--) {
- final DatanodeDescriptor dn = currentBlock.getDatanode(i);
- final DatanodeStorageInfo storage = currentBlock.findStorageInfo(dn);
- final boolean removed = storage.removeBlock(currentBlock);
- Preconditions.checkState(removed, "currentBlock not found.");
-
- final AddBlockResult result = storage.addBlock(newBlock);
- Preconditions.checkState(result == AddBlockResult.ADDED,
- "newBlock already exists.");
- }
+ currentBlock.replaceBlock(newBlock);
// replace block in the map itself
blocks.put(newBlock);
return newBlock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 9acd81f..3de6760 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
@@ -1055,7 +1056,7 @@ public class FSEditLogLoader {
// is only executed when loading edits written by prior
// versions of Hadoop. Current versions always log
// OP_ADD operations as each block is allocated.
- newBI = new BlockInfo(newBlock,
+ newBI = new BlockInfoContiguous(newBlock,
file.getPreferredBlockReplication());
}
fsNamesys.getBlockManager().addBlockCollection(newBI, file);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index a46f2b6..30517d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.protocol.LayoutFlags;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -755,7 +756,7 @@ public class FSImageFormat {
// read blocks
BlockInfo[] blocks = new BlockInfo[numBlocks];
for (int j = 0; j < numBlocks; j++) {
- blocks[j] = new BlockInfo(replication);
+ blocks[j] = new BlockInfoContiguous(replication);
blocks[j].readFields(in);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 29713a2..1c14220 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
@@ -325,7 +326,8 @@ public final class FSImageFormatPBINode {
BlockInfo[] blocks = new BlockInfo[bp.size()];
for (int i = 0, e = bp.size(); i < e; ++i) {
- blocks[i] = new BlockInfo(PBHelper.convert(bp.get(i)), replication);
+ blocks[i] =
+ new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
}
final PermissionStatus permissions = loadPermission(f.getPermission(),
parent.getLoaderContext().getStringTable());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index d790b8f..f71cf0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat;
@@ -131,7 +132,7 @@ public class FSImageSerialization {
int i = 0;
for (; i < numBlocks-1; i++) {
blk.readFields(in);
- blocks[i] = new BlockInfo(blk, blockReplication);
+ blocks[i] = new BlockInfoContiguous(blk, blockReplication);
}
// last block is UNDER_CONSTRUCTION
if(numBlocks > 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index 363c208..64ad1f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.namenode.AclEntryStatusFormat;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -245,7 +246,7 @@ public class FSImageFormatPBSnapshot {
BlockInfo storedBlock = fsn.getBlockManager().getStoredBlock(blk);
if(storedBlock == null) {
storedBlock = fsn.getBlockManager().addBlockCollection(
- new BlockInfo(blk, copy.getFileReplication()), file);
+ new BlockInfoContiguous(blk, copy.getFileReplication()), file);
}
blocks[j] = storedBlock;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index 96d4d8b..5126aa7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -49,7 +49,7 @@ public class TestBlockInfo {
@Test
public void testIsDeleted() {
- BlockInfo blockInfo = new BlockInfo((short) 3);
+ BlockInfo blockInfo = new BlockInfoContiguous((short) 3);
BlockCollection bc = Mockito.mock(BlockCollection.class);
blockInfo.setBlockCollection(bc);
Assert.assertFalse(blockInfo.isDeleted());
@@ -59,7 +59,7 @@ public class TestBlockInfo {
@Test
public void testAddStorage() throws Exception {
- BlockInfo blockInfo = new BlockInfo((short) 3);
+ BlockInfo blockInfo = new BlockInfoContiguous((short) 3);
final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1");
@@ -71,9 +71,9 @@ public class TestBlockInfo {
@Test
public void testCopyConstructor() {
- BlockInfo old = new BlockInfo((short) 3);
+ BlockInfo old = new BlockInfoContiguous((short) 3);
try {
- BlockInfo copy = new BlockInfo(old);
+ BlockInfo copy = new BlockInfoContiguous((BlockInfoContiguous)old);
assertEquals(old.getBlockCollection(), copy.getBlockCollection());
assertEquals(old.getCapacity(), copy.getCapacity());
} catch (Exception e) {
@@ -92,7 +92,7 @@ public class TestBlockInfo {
// Create a few dummy blocks and add them to the first storage.
for (int i = 0; i < NUM_BLOCKS; ++i) {
- blockInfos[i] = new BlockInfo((short) 3);
+ blockInfos[i] = new BlockInfoContiguous((short) 3);
storage1.addBlock(blockInfos[i]);
}
@@ -118,7 +118,7 @@ public class TestBlockInfo {
LOG.info("Building block list...");
for (int i = 0; i < MAX_BLOCKS; i++) {
blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
- blockInfoList.add(new BlockInfo(blockList.get(i), (short) 3));
+ blockInfoList.add(new BlockInfoContiguous(blockList.get(i), (short) 3));
dd.addBlock(blockInfoList.get(i));
// index of the datanode should be 0
@@ -182,4 +182,4 @@ public class TestBlockInfo {
blockInfoList.get(j), dd.getBlockListHeadForTesting());
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index bf46ed7..fd03745 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -389,7 +389,7 @@ public class TestBlockManager {
private BlockInfo blockOnNodes(long blkId, List<DatanodeDescriptor> nodes) {
Block block = new Block(blkId);
- BlockInfo blockInfo = new BlockInfo(block, (short) 3);
+ BlockInfo blockInfo = new BlockInfoContiguous(block, (short) 3);
for (DatanodeDescriptor dn : nodes) {
for (DatanodeStorageInfo storage : dn.getStorageInfos()) {
@@ -738,7 +738,7 @@ public class TestBlockManager {
private BlockInfo addBlockToBM(long blkId) {
Block block = new Block(blkId);
BlockInfo blockInfo =
- new BlockInfo(block, (short) 3);
+ new BlockInfoContiguous(block, (short) 3);
BlockCollection bc = Mockito.mock(BlockCollection.class);
Mockito.doReturn((short) 3).when(bc).getPreferredBlockReplication();
bm.blocksMap.addBlockCollection(blockInfo, bc);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
index fe639e4..ca27bb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
@@ -58,8 +58,8 @@ public class TestDatanodeDescriptor {
public void testBlocksCounter() throws Exception {
DatanodeDescriptor dd = BlockManagerTestUtil.getLocalDatanodeDescriptor(true);
assertEquals(0, dd.numBlocks());
- BlockInfo blk = new BlockInfo(new Block(1L), (short) 1);
- BlockInfo blk1 = new BlockInfo(new Block(2L), (short) 2);
+ BlockInfo blk = new BlockInfoContiguous(new Block(1L), (short) 1);
+ BlockInfo blk1 = new BlockInfoContiguous(new Block(2L), (short) 2);
DatanodeStorageInfo[] storages = dd.getStorageInfos();
assertTrue(storages.length > 0);
// add first block
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
index 844c5d0..090c6e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
@@ -178,7 +178,7 @@ public class TestPendingReplication {
//
block = new Block(1, 1, 0);
- blockInfo = new BlockInfo(block, (short) 3);
+ blockInfo = new BlockInfoContiguous(block, (short) 3);
pendingReplications.increment(block,
DatanodeStorageInfo.toDatanodeDescriptors(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index f317723..fccaf3c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1218,7 +1218,7 @@ public class TestReplicationPolicy {
chosenBlocks = underReplicatedBlocks.chooseUnderReplicatedBlocks(1);
assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
- final BlockInfo info = new BlockInfo(block1, (short) 1);
+ final BlockInfo info = new BlockInfoContiguous(block1, (short) 1);
final BlockCollection mbc = mock(BlockCollection.class);
when(mbc.getLastBlock()).thenReturn(info);
when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index 3f96c0c..8a606f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.Storage;
@@ -69,8 +70,8 @@ public class CreateEditsLog {
BlockInfo[] blocks = new BlockInfo[blocksPerFile];
for (int iB = 0; iB < blocksPerFile; ++iB) {
blocks[iB] =
- new BlockInfo(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
- replication);
+ new BlockInfoContiguous(new Block(0, blockSize, BLOCK_GENERATION_STAMP),
+ replication);
}
long currentBlockId = startingBlockId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index 8c9da01..bea7241 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -106,7 +107,7 @@ public class TestCommitBlockSynchronization {
lastBlock, genStamp, length, false, false, newTargets, null);
// Simulate 'completing' the block.
- BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
+ BlockInfo completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
completedBlockInfo.setBlockCollection(file);
completedBlockInfo.setGenerationStamp(genStamp);
doReturn(completedBlockInfo).when(namesystemSpy)
@@ -178,7 +179,7 @@ public class TestCommitBlockSynchronization {
namesystemSpy.commitBlockSynchronization(
lastBlock, genStamp, length, true, false, newTargets, null);
- BlockInfo completedBlockInfo = new BlockInfo(block, (short) 1);
+ BlockInfo completedBlockInfo = new BlockInfoContiguous(block, (short) 1);
completedBlockInfo.setBlockCollection(file);
completedBlockInfo.setGenerationStamp(genStamp);
doReturn(completedBlockInfo).when(namesystemSpy)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index 21b67a7..ec4de1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.IOUtils;
@@ -290,7 +291,7 @@ public class TestINodeFile {
iNodes[i] = new INodeFile(i, null, perm, 0L, 0L, null, replication,
preferredBlockSize);
iNodes[i].setLocalName(DFSUtil.string2Bytes(fileNamePrefix + i));
- BlockInfo newblock = new BlockInfo(replication);
+ BlockInfo newblock = new BlockInfoContiguous(replication);
iNodes[i].addBlock(newblock);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
index 0d726e9..106edad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
@@ -148,7 +149,7 @@ public class TestTruncateQuotaUpdate {
private BlockInfo newBlock(long size, short replication) {
Block b = new Block(++nextMockBlockId, size, ++nextMockGenstamp);
- return new BlockInfo(b, replication);
+ return new BlockInfoContiguous(b, replication);
}
private static void addSnapshotFeature(INodeFile file, BlockInfo[] blocks) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/065ab4d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
index ac81488..0d6298f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
@@ -21,6 +21,7 @@ import com.google.common.collect.Lists;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
@@ -50,7 +51,7 @@ public class TestFileWithSnapshotFeature {
BlockStoragePolicySuite bsps = mock(BlockStoragePolicySuite.class);
BlockStoragePolicy bsp = mock(BlockStoragePolicy.class);
BlockInfo[] blocks = new BlockInfo[] {
- new BlockInfo(new Block(1, BLOCK_SIZE, 1), REPL_1)
+ new BlockInfoContiguous(new Block(1, BLOCK_SIZE, 1), REPL_1)
};
// No snapshot
[46/50] [abbrv] hadoop git commit: HADOOP-9905. remove dependency of
zookeeper for hadoop-client (Contributed by Vinayakumar B)
Posted by zj...@apache.org.
HADOOP-9905. remove dependency of zookeeper for hadoop-client (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/850e4888
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/850e4888
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/850e4888
Branch: refs/heads/YARN-2928
Commit: 850e48887c143c28f3a1682f527ea59cf5c38560
Parents: 732a70f
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Jun 1 17:47:02 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:22 2015 -0700
----------------------------------------------------------------------
hadoop-client/pom.xml | 8 ++++++++
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/850e4888/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml
index f8b6d97..2324dba 100644
--- a/hadoop-client/pom.xml
+++ b/hadoop-client/pom.xml
@@ -95,6 +95,10 @@
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -171,6 +175,10 @@
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/850e4888/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 00075b4..44c2969 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -480,6 +480,8 @@ Trunk (Unreleased)
HADOOP-12022. fix site -Pdocs -Pdist in hadoop-project-dist; cleanout
remaining forrest bits (aw)
+ HADOOP-9905. remove dependency of zookeeper for hadoop-client (vinayakumarb)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
[23/50] [abbrv] hadoop git commit: HADOOP-11934. Use of
JavaKeyStoreProvider in LdapGroupsMapping causes infinite loop. Contributed
by Larry McCay.
Posted by zj...@apache.org.
HADOOP-11934. Use of JavaKeyStoreProvider in LdapGroupsMapping causes infinite loop. Contributed by Larry McCay.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3fcb33b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3fcb33b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3fcb33b3
Branch: refs/heads/YARN-2928
Commit: 3fcb33b37d05bd6af730fd6dfbc054c2686a7553
Parents: b17498d
Author: cnauroth <cn...@apache.org>
Authored: Thu May 28 15:01:42 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../alias/AbstractJavaKeyStoreProvider.java | 344 +++++++++++++++++++
.../security/alias/JavaKeyStoreProvider.java | 257 ++------------
.../alias/LocalJavaKeyStoreProvider.java | 192 +++++++++++
...oop.security.alias.CredentialProviderFactory | 1 +
.../alias/TestCredentialProviderFactory.java | 25 +-
6 files changed, 597 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 71b71a0..af82c0b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -832,6 +832,9 @@ Release 2.7.1 - UNRELEASED
HADOOP-11973. Ensure ZkDelegationTokenSecretManager namespace znodes get
created with ACLs. (Gregory Chanan via asuresh)
+ HADOOP-11934. Use of JavaKeyStoreProvider in LdapGroupsMapping causes
+ infinite loop. (Larry McCay via cnauroth)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
new file mode 100644
index 0000000..9251044
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
+
+import com.google.common.base.Charsets;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Abstract class for implementing credential providers that are based on
+ * Java Keystores as the underlying credential store.
+ *
+ * The password for the keystore is taken from the HADOOP_CREDSTORE_PASSWORD
+ * environment variable with a default of 'none'.
+ *
+ * It is expected that for access to credential protected resource to copy the
+ * creds from the original provider into the job's Credentials object, which is
+ * accessed via the UserProvider. Therefore, these providers won't be directly
+ * used by MapReduce tasks.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
+ public static final String CREDENTIAL_PASSWORD_NAME =
+ "HADOOP_CREDSTORE_PASSWORD";
+ public static final String KEYSTORE_PASSWORD_FILE_KEY =
+ "hadoop.security.credstore.java-keystore-provider.password-file";
+ public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
+
+ private Path path;
+ private final URI uri;
+ private final KeyStore keyStore;
+ private char[] password = null;
+ private boolean changed = false;
+ private Lock readLock;
+ private Lock writeLock;
+
+ protected AbstractJavaKeyStoreProvider(URI uri, Configuration conf)
+ throws IOException {
+ this.uri = uri;
+ initFileSystem(uri, conf);
+ // Get the password from the user's environment
+ if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
+ password = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
+ }
+ // if not in ENV get check for file
+ if (password == null) {
+ String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
+ if (pwFile != null) {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ URL pwdFile = cl.getResource(pwFile);
+ if (pwdFile != null) {
+ try (InputStream is = pwdFile.openStream()) {
+ password = IOUtils.toString(is).trim().toCharArray();
+ }
+ }
+ }
+ }
+ if (password == null) {
+ password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
+ }
+ try {
+ keyStore = KeyStore.getInstance("jceks");
+ if (keystoreExists()) {
+ stashOriginalFilePermissions();
+ try (InputStream in = getInputStreamForFile()) {
+ keyStore.load(in, password);
+ }
+ } else {
+ createPermissions("700");
+ // required to create an empty keystore. *sigh*
+ keyStore.load(null, password);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't create keystore", e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException("Can't load keystore " + getPathAsString(), e);
+ } catch (CertificateException e) {
+ throw new IOException("Can't load keystore " + getPathAsString(), e);
+ }
+ ReadWriteLock lock = new ReentrantReadWriteLock(true);
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+ }
+
+ public Path getPath() {
+ return path;
+ }
+
+ public void setPath(Path p) {
+ this.path = p;
+ }
+
+ public char[] getPassword() {
+ return password;
+ }
+
+ public void setPassword(char[] pass) {
+ this.password = pass;
+ }
+
+ public boolean isChanged() {
+ return changed;
+ }
+
+ public void setChanged(boolean chg) {
+ this.changed = chg;
+ }
+
+ public Lock getReadLock() {
+ return readLock;
+ }
+
+ public void setReadLock(Lock rl) {
+ this.readLock = rl;
+ }
+
+ public Lock getWriteLock() {
+ return writeLock;
+ }
+
+ public void setWriteLock(Lock wl) {
+ this.writeLock = wl;
+ }
+
+ public URI getUri() {
+ return uri;
+ }
+
+ public KeyStore getKeyStore() {
+ return keyStore;
+ }
+
+ public Map<String, CredentialEntry> getCache() {
+ return cache;
+ }
+
+ private final Map<String, CredentialEntry> cache =
+ new HashMap<String, CredentialEntry>();
+
+ protected final String getPathAsString() {
+ return getPath().toString();
+ }
+
+ protected abstract String getSchemeName();
+
+ protected abstract OutputStream getOutputStreamForKeystore()
+ throws IOException;
+
+ protected abstract boolean keystoreExists() throws IOException;
+
+ protected abstract InputStream getInputStreamForFile() throws IOException;
+
+ protected abstract void createPermissions(String perms) throws IOException;
+
+ protected abstract void stashOriginalFilePermissions() throws IOException;
+
+ protected void initFileSystem(URI keystoreUri, Configuration conf)
+ throws IOException {
+ path = ProviderUtils.unnestUri(keystoreUri);
+ }
+
+ @Override
+ public CredentialEntry getCredentialEntry(String alias)
+ throws IOException {
+ readLock.lock();
+ try {
+ SecretKeySpec key = null;
+ try {
+ if (cache.containsKey(alias)) {
+ return cache.get(alias);
+ }
+ if (!keyStore.containsAlias(alias)) {
+ return null;
+ }
+ key = (SecretKeySpec) keyStore.getKey(alias, password);
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't get credential " + alias + " from "
+ + getPathAsString(), e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException("Can't get algorithm for credential " + alias
+ + " from " + getPathAsString(), e);
+ } catch (UnrecoverableKeyException e) {
+ throw new IOException("Can't recover credential " + alias + " from "
+ + getPathAsString(), e);
+ }
+ return new CredentialEntry(alias, bytesToChars(key.getEncoded()));
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public static char[] bytesToChars(byte[] bytes) throws IOException {
+ String pass;
+ pass = new String(bytes, Charsets.UTF_8);
+ return pass.toCharArray();
+ }
+
+ @Override
+ public List<String> getAliases() throws IOException {
+ readLock.lock();
+ try {
+ ArrayList<String> list = new ArrayList<String>();
+ String alias = null;
+ try {
+ Enumeration<String> e = keyStore.aliases();
+ while (e.hasMoreElements()) {
+ alias = e.nextElement();
+ list.add(alias);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't get alias " + alias + " from "
+ + getPathAsString(), e);
+ }
+ return list;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public CredentialEntry createCredentialEntry(String alias, char[] credential)
+ throws IOException {
+ writeLock.lock();
+ try {
+ if (keyStore.containsAlias(alias) || cache.containsKey(alias)) {
+ throw new IOException("Credential " + alias + " already exists in "
+ + this);
+ }
+ return innerSetCredential(alias, credential);
+ } catch (KeyStoreException e) {
+ throw new IOException("Problem looking up credential " + alias + " in "
+ + this, e);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void deleteCredentialEntry(String name) throws IOException {
+ writeLock.lock();
+ try {
+ try {
+ if (keyStore.containsAlias(name)) {
+ keyStore.deleteEntry(name);
+ } else {
+ throw new IOException("Credential " + name + " does not exist in "
+ + this);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Problem removing " + name + " from " + this, e);
+ }
+ cache.remove(name);
+ changed = true;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ CredentialEntry innerSetCredential(String alias, char[] material)
+ throws IOException {
+ writeLock.lock();
+ try {
+ keyStore.setKeyEntry(alias,
+ new SecretKeySpec(new String(material).getBytes("UTF-8"), "AES"),
+ password, null);
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't store credential " + alias + " in " + this,
+ e);
+ } finally {
+ writeLock.unlock();
+ }
+ changed = true;
+ return new CredentialEntry(alias, material);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ writeLock.lock();
+ try {
+ if (!changed) {
+ return;
+ }
+ // write out the keystore
+ try (OutputStream out = getOutputStreamForKeystore()) {
+ keyStore.store(out, password);
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't store keystore " + this, e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException("No such algorithm storing keystore " + this, e);
+ } catch (CertificateException e) {
+ throw new IOException("Certificate exception storing keystore " + this,
+ e);
+ }
+ changed = false;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return uri.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
index 5e5cebb..d02ebde 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
@@ -18,266 +18,75 @@
package org.apache.hadoop.security.alias;
-import org.apache.commons.io.Charsets;
-import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.ProviderUtils;
-import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.net.URI;
-import java.net.URL;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.UnrecoverableKeyException;
-import java.security.cert.CertificateException;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
- * CredentialProvider based on Java's KeyStore file format. The file may be
+ * CredentialProvider based on Java's KeyStore file format. The file may be
* stored in any Hadoop FileSystem using the following name mangling:
- * jceks://hdfs@nn1.example.com/my/creds.jceks -> hdfs://nn1.example.com/my/creds.jceks
- * jceks://file/home/larry/creds.jceks -> file:///home/larry/creds.jceks
- *
- * The password for the keystore is taken from the HADOOP_CREDSTORE_PASSWORD
- * environment variable with a default of 'none'.
- *
- * It is expected that for access to credential protected resource to copy the
- * creds from the original provider into the job's Credentials object, which is
- * accessed via the UserProvider. Therefore, this provider won't be directly
- * used by MapReduce tasks.
+ * jceks://hdfs@nn1.example.com/my/creds.jceks ->
+ * hdfs://nn1.example.com/my/creds.jceks jceks://file/home/larry/creds.jceks ->
+ * file:///home/larry/creds.jceks
*/
@InterfaceAudience.Private
-public class JavaKeyStoreProvider extends CredentialProvider {
+public class JavaKeyStoreProvider extends AbstractJavaKeyStoreProvider {
public static final String SCHEME_NAME = "jceks";
- public static final String CREDENTIAL_PASSWORD_NAME =
- "HADOOP_CREDSTORE_PASSWORD";
- public static final String KEYSTORE_PASSWORD_FILE_KEY =
- "hadoop.security.credstore.java-keystore-provider.password-file";
- public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
-
- private final URI uri;
- private final Path path;
- private final FileSystem fs;
- private final FsPermission permissions;
- private final KeyStore keyStore;
- private char[] password = null;
- private boolean changed = false;
- private Lock readLock;
- private Lock writeLock;
-
- private final Map<String, CredentialEntry> cache = new HashMap<String, CredentialEntry>();
-
- private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
- this.uri = uri;
- path = ProviderUtils.unnestUri(uri);
- fs = path.getFileSystem(conf);
- // Get the password from the user's environment
- if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
- password = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
- }
- // if not in ENV get check for file
- if (password == null) {
- String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
- if (pwFile != null) {
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
- URL pwdFile = cl.getResource(pwFile);
- if (pwdFile != null) {
- try (InputStream is = pwdFile.openStream()) {
- password = IOUtils.toString(is).trim().toCharArray();
- }
- }
- }
- }
- if (password == null) {
- password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
- }
- try {
- keyStore = KeyStore.getInstance(SCHEME_NAME);
- if (fs.exists(path)) {
- // save off permissions in case we need to
- // rewrite the keystore in flush()
- FileStatus s = fs.getFileStatus(path);
- permissions = s.getPermission();
+ private FileSystem fs;
+ private FsPermission permissions;
- try (FSDataInputStream in = fs.open(path)) {
- keyStore.load(in, password);
- }
- } else {
- permissions = new FsPermission("700");
- // required to create an empty keystore. *sigh*
- keyStore.load(null, password);
- }
- } catch (KeyStoreException e) {
- throw new IOException("Can't create keystore", e);
- } catch (NoSuchAlgorithmException e) {
- throw new IOException("Can't load keystore " + path, e);
- } catch (CertificateException e) {
- throw new IOException("Can't load keystore " + path, e);
- }
- ReadWriteLock lock = new ReentrantReadWriteLock(true);
- readLock = lock.readLock();
- writeLock = lock.writeLock();
+ private JavaKeyStoreProvider(URI uri, Configuration conf)
+ throws IOException {
+ super(uri, conf);
}
@Override
- public CredentialEntry getCredentialEntry(String alias) throws IOException {
- readLock.lock();
- try {
- SecretKeySpec key = null;
- try {
- if (cache.containsKey(alias)) {
- return cache.get(alias);
- }
- if (!keyStore.containsAlias(alias)) {
- return null;
- }
- key = (SecretKeySpec) keyStore.getKey(alias, password);
- } catch (KeyStoreException e) {
- throw new IOException("Can't get credential " + alias + " from " +
- path, e);
- } catch (NoSuchAlgorithmException e) {
- throw new IOException("Can't get algorithm for credential " + alias + " from " +
- path, e);
- } catch (UnrecoverableKeyException e) {
- throw new IOException("Can't recover credential " + alias + " from " + path, e);
- }
- return new CredentialEntry(alias, bytesToChars(key.getEncoded()));
- }
- finally {
- readLock.unlock();
- }
- }
-
- public static char[] bytesToChars(byte[] bytes) {
- String pass = new String(bytes, Charsets.UTF_8);
- return pass.toCharArray();
+ protected String getSchemeName() {
+ return SCHEME_NAME;
}
@Override
- public List<String> getAliases() throws IOException {
- readLock.lock();
- try {
- ArrayList<String> list = new ArrayList<String>();
- String alias = null;
- try {
- Enumeration<String> e = keyStore.aliases();
- while (e.hasMoreElements()) {
- alias = e.nextElement();
- list.add(alias);
- }
- } catch (KeyStoreException e) {
- throw new IOException("Can't get alias " + alias + " from " + path, e);
- }
- return list;
- }
- finally {
- readLock.unlock();
- }
+ protected OutputStream getOutputStreamForKeystore() throws IOException {
+ FSDataOutputStream out = FileSystem.create(fs, getPath(), permissions);
+ return out;
}
@Override
- public CredentialEntry createCredentialEntry(String alias, char[] credential)
- throws IOException {
- writeLock.lock();
- try {
- if (keyStore.containsAlias(alias) || cache.containsKey(alias)) {
- throw new IOException("Credential " + alias + " already exists in " + this);
- }
- return innerSetCredential(alias, credential);
- } catch (KeyStoreException e) {
- throw new IOException("Problem looking up credential " + alias + " in " + this,
- e);
- } finally {
- writeLock.unlock();
- }
+ protected boolean keystoreExists() throws IOException {
+ return fs.exists(getPath());
}
@Override
- public void deleteCredentialEntry(String name) throws IOException {
- writeLock.lock();
- try {
- try {
- if (keyStore.containsAlias(name)) {
- keyStore.deleteEntry(name);
- }
- else {
- throw new IOException("Credential " + name + " does not exist in " + this);
- }
- } catch (KeyStoreException e) {
- throw new IOException("Problem removing " + name + " from " +
- this, e);
- }
- cache.remove(name);
- changed = true;
- }
- finally {
- writeLock.unlock();
- }
+ protected InputStream getInputStreamForFile() throws IOException {
+ return fs.open(getPath());
}
- CredentialEntry innerSetCredential(String alias, char[] material)
- throws IOException {
- writeLock.lock();
- try {
- keyStore.setKeyEntry(alias, new SecretKeySpec(
- new String(material).getBytes("UTF-8"), "AES"),
- password, null);
- } catch (KeyStoreException e) {
- throw new IOException("Can't store credential " + alias + " in " + this,
- e);
- } finally {
- writeLock.unlock();
- }
- changed = true;
- return new CredentialEntry(alias, material);
+ @Override
+ protected void createPermissions(String perms) {
+ permissions = new FsPermission(perms);
}
@Override
- public void flush() throws IOException {
- writeLock.lock();
- try {
- if (!changed) {
- return;
- }
- // write out the keystore
- try (FSDataOutputStream out = FileSystem.create(fs, path, permissions)) {
- keyStore.store(out, password);
- } catch (KeyStoreException e) {
- throw new IOException("Can't store keystore " + this, e);
- } catch (NoSuchAlgorithmException e) {
- throw new IOException("No such algorithm storing keystore " + this, e);
- } catch (CertificateException e) {
- throw new IOException("Certificate exception storing keystore " + this,
- e);
- }
- changed = false;
- }
- finally {
- writeLock.unlock();
- }
+ protected void stashOriginalFilePermissions() throws IOException {
+ // save off permissions in case we need to
+ // rewrite the keystore in flush()
+ FileStatus s = fs.getFileStatus(getPath());
+ permissions = s.getPermission();
}
- @Override
- public String toString() {
- return uri.toString();
+ protected void initFileSystem(URI uri, Configuration conf)
+ throws IOException {
+ super.initFileSystem(uri, conf);
+ fs = getPath().getFileSystem(conf);
}
/**
@@ -286,7 +95,7 @@ public class JavaKeyStoreProvider extends CredentialProvider {
public static class Factory extends CredentialProviderFactory {
@Override
public CredentialProvider createProvider(URI providerName,
- Configuration conf) throws IOException {
+ Configuration conf) throws IOException {
if (SCHEME_NAME.equals(providerName.getScheme())) {
return new JavaKeyStoreProvider(providerName, conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
new file mode 100644
index 0000000..3840979
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.EnumSet;
+
+/**
+ * CredentialProvider based on Java's KeyStore file format. The file may be
+ * stored only on the local filesystem using the following name mangling:
+ * localjceks://file/home/larry/creds.jceks -> file:///home/larry/creds.jceks
+ */
+@InterfaceAudience.Private
+public final class LocalJavaKeyStoreProvider extends
+ AbstractJavaKeyStoreProvider {
+ public static final String SCHEME_NAME = "localjceks";
+ private File file;
+ private Set<PosixFilePermission> permissions;
+
+ private LocalJavaKeyStoreProvider(URI uri, Configuration conf)
+ throws IOException {
+ super(uri, conf);
+ }
+
+ @Override
+ protected String getSchemeName() {
+ return SCHEME_NAME;
+ }
+
+ @Override
+ protected OutputStream getOutputStreamForKeystore() throws IOException {
+ FileOutputStream out = new FileOutputStream(file);
+ return out;
+ }
+
+ @Override
+ protected boolean keystoreExists() throws IOException {
+ return file.exists();
+ }
+
+ @Override
+ protected InputStream getInputStreamForFile() throws IOException {
+ FileInputStream is = new FileInputStream(file);
+ return is;
+ }
+
+ @Override
+ protected void createPermissions(String perms) throws IOException {
+ int mode = 700;
+ try {
+ mode = Integer.parseInt(perms, 8);
+ } catch (NumberFormatException nfe) {
+ throw new IOException("Invalid permissions mode provided while "
+ + "trying to createPermissions", nfe);
+ }
+ permissions = modeToPosixFilePermission(mode);
+ }
+
+ @Override
+ protected void stashOriginalFilePermissions() throws IOException {
+ // save off permissions in case we need to
+ // rewrite the keystore in flush()
+ if (!Shell.WINDOWS) {
+ Path path = Paths.get(file.getCanonicalPath());
+ permissions = Files.getPosixFilePermissions(path);
+ } else {
+ // On Windows, the JDK does not support the POSIX file permission APIs.
+ // Instead, we can do a winutils call and translate.
+ String[] cmd = Shell.getGetPermissionCommand();
+ String[] args = new String[cmd.length + 1];
+ System.arraycopy(cmd, 0, args, 0, cmd.length);
+ args[cmd.length] = file.getCanonicalPath();
+ String out = Shell.execCommand(args);
+ StringTokenizer t = new StringTokenizer(out, Shell.TOKEN_SEPARATOR_REGEX);
+ // The winutils output consists of 10 characters because of the leading
+ // directory indicator, i.e. "drwx------". The JDK parsing method expects
+ // a 9-character string, so remove the leading character.
+ String permString = t.nextToken().substring(1);
+ permissions = PosixFilePermissions.fromString(permString);
+ }
+ }
+
+ @Override
+ protected void initFileSystem(URI uri, Configuration conf)
+ throws IOException {
+ super.initFileSystem(uri, conf);
+ try {
+ file = new File(new URI(getPath().toString()));
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ super.flush();
+ if (!Shell.WINDOWS) {
+ Files.setPosixFilePermissions(Paths.get(file.getCanonicalPath()),
+ permissions);
+ } else {
+ // FsPermission expects a 10-character string because of the leading
+ // directory indicator, i.e. "drwx------". The JDK toString method returns
+ // a 9-character string, so prepend a leading character.
+ FsPermission fsPermission = FsPermission.valueOf(
+ "-" + PosixFilePermissions.toString(permissions));
+ FileUtil.setPermission(file, fsPermission);
+ }
+ }
+
+ /**
+ * The factory to create JksProviders, which is used by the ServiceLoader.
+ */
+ public static class Factory extends CredentialProviderFactory {
+ @Override
+ public CredentialProvider createProvider(URI providerName,
+ Configuration conf) throws IOException {
+ if (SCHEME_NAME.equals(providerName.getScheme())) {
+ return new LocalJavaKeyStoreProvider(providerName, conf);
+ }
+ return null;
+ }
+ }
+
+ private static Set<PosixFilePermission> modeToPosixFilePermission(
+ int mode) {
+ Set<PosixFilePermission> perms = EnumSet.noneOf(PosixFilePermission.class);
+ if ((mode & 0001) != 0) {
+ perms.add(PosixFilePermission.OTHERS_EXECUTE);
+ }
+ if ((mode & 0002) != 0) {
+ perms.add(PosixFilePermission.OTHERS_WRITE);
+ }
+ if ((mode & 0004) != 0) {
+ perms.add(PosixFilePermission.OTHERS_READ);
+ }
+ if ((mode & 0010) != 0) {
+ perms.add(PosixFilePermission.GROUP_EXECUTE);
+ }
+ if ((mode & 0020) != 0) {
+ perms.add(PosixFilePermission.GROUP_WRITE);
+ }
+ if ((mode & 0040) != 0) {
+ perms.add(PosixFilePermission.GROUP_READ);
+ }
+ if ((mode & 0100) != 0) {
+ perms.add(PosixFilePermission.OWNER_EXECUTE);
+ }
+ if ((mode & 0200) != 0) {
+ perms.add(PosixFilePermission.OWNER_WRITE);
+ }
+ if ((mode & 0400) != 0) {
+ perms.add(PosixFilePermission.OWNER_READ);
+ }
+ return perms;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
index bb7366e..f673cf4 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
@@ -14,4 +14,5 @@
# limitations under the License.
org.apache.hadoop.security.alias.JavaKeyStoreProvider$Factory
+org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider$Factory
org.apache.hadoop.security.alias.UserProvider$Factory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fcb33b3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
index 1eb1e8b..16cb0be 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
@@ -207,7 +207,29 @@ public class TestCredentialProviderFactory {
checkPermissionRetention(conf, ourUrl, path);
}
- public void checkPermissionRetention(Configuration conf, String ourUrl,
+ @Test
+ public void testLocalJksProvider() throws Exception {
+ Configuration conf = new Configuration();
+ final Path jksPath = new Path(tmpDir.toString(), "test.jks");
+ final String ourUrl =
+ LocalJavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+ File file = new File(tmpDir, "test.jks");
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+ checkSpecificProvider(conf, ourUrl);
+ Path path = ProviderUtils.unnestUri(new URI(ourUrl));
+ FileSystem fs = path.getFileSystem(conf);
+ FileStatus s = fs.getFileStatus(path);
+ assertTrue("Unexpected permissions: " + s.getPermission().toString(), s.getPermission().toString().equals("rwx------"));
+ assertTrue(file + " should exist", file.isFile());
+
+ // check permission retention after explicit change
+ fs.setPermission(path, new FsPermission("777"));
+ checkPermissionRetention(conf, ourUrl, path);
+ }
+
+ public void checkPermissionRetention(Configuration conf, String ourUrl,
Path path) throws Exception {
CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0);
// let's add a new credential and flush and check that permissions are still set to 777
@@ -233,3 +255,4 @@ public class TestCredentialProviderFactory {
"keystore.", s.getPermission().toString().equals("rwxrwxrwx"));
}
}
+
[03/50] [abbrv] hadoop git commit: HDFS-8482. Rename
BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
index f6b18e6..0d726e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTruncateQuotaUpdate.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
@@ -100,7 +100,7 @@ public class TestTruncateQuotaUpdate {
@Test
public void testTruncateWithSnapshotAndDivergence() {
INodeFile file = createMockFile(BLOCKSIZE * 2 + BLOCKSIZE / 2, REPLICATION);
- BlockInfoContiguous[] blocks = new BlockInfoContiguous
+ BlockInfo[] blocks = new BlockInfo
[file.getBlocks().length];
System.arraycopy(file.getBlocks(), 0, blocks, 0, blocks.length);
addSnapshotFeature(file, blocks);
@@ -130,11 +130,11 @@ public class TestTruncateQuotaUpdate {
}
private INodeFile createMockFile(long size, short replication) {
- ArrayList<BlockInfoContiguous> blocks = new ArrayList<>();
+ ArrayList<BlockInfo> blocks = new ArrayList<>();
long createdSize = 0;
while (createdSize < size) {
long blockSize = Math.min(BLOCKSIZE, size - createdSize);
- BlockInfoContiguous bi = newBlock(blockSize, replication);
+ BlockInfo bi = newBlock(blockSize, replication);
blocks.add(bi);
createdSize += BLOCKSIZE;
}
@@ -142,16 +142,16 @@ public class TestTruncateQuotaUpdate {
.createImmutable((short) 0x1ff));
return new INodeFile(
++nextMockINodeId, new byte[0], perm, 0, 0,
- blocks.toArray(new BlockInfoContiguous[blocks.size()]), replication,
+ blocks.toArray(new BlockInfo[blocks.size()]), replication,
BLOCKSIZE);
}
- private BlockInfoContiguous newBlock(long size, short replication) {
+ private BlockInfo newBlock(long size, short replication) {
Block b = new Block(++nextMockBlockId, size, ++nextMockGenstamp);
- return new BlockInfoContiguous(b, replication);
+ return new BlockInfo(b, replication);
}
- private static void addSnapshotFeature(INodeFile file, BlockInfoContiguous[] blocks) {
+ private static void addSnapshotFeature(INodeFile file, BlockInfo[] blocks) {
FileDiff diff = mock(FileDiff.class);
when(diff.getBlocks()).thenReturn(blocks);
FileDiffList diffList = new FileDiffList();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
index 11b19f3..a1abd08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceStorage;
@@ -176,7 +176,7 @@ public class SnapshotTestHelper {
*
* Specific information for different types of INode:
* {@link INodeDirectory}:childrenSize
- * {@link INodeFile}: fileSize, block list. Check {@link BlockInfoContiguous#toString()}
+ * {@link INodeFile}: fileSize, block list. Check {@link BlockInfo#toString()}
* and {@link BlockInfoContiguousUnderConstruction#toString()} for detailed information.
* {@link FileWithSnapshot}: next link
* </pre>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
index 8b9ebea..ac81488 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
import com.google.common.collect.Lists;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
@@ -49,8 +49,8 @@ public class TestFileWithSnapshotFeature {
FileDiff diff = mock(FileDiff.class);
BlockStoragePolicySuite bsps = mock(BlockStoragePolicySuite.class);
BlockStoragePolicy bsp = mock(BlockStoragePolicy.class);
- BlockInfoContiguous[] blocks = new BlockInfoContiguous[] {
- new BlockInfoContiguous(new Block(1, BLOCK_SIZE, 1), REPL_1)
+ BlockInfo[] blocks = new BlockInfo[] {
+ new BlockInfo(new Block(1, BLOCK_SIZE, 1), REPL_1)
};
// No snapshot
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
index 85072d1..7bffb33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -108,14 +108,14 @@ public class TestSnapshotBlocksMap {
final FSDirectory dir, final BlockManager blkManager) throws Exception {
final INodeFile file = INodeFile.valueOf(dir.getINode(path), path);
assertEquals(numBlocks, file.getBlocks().length);
- for(BlockInfoContiguous b : file.getBlocks()) {
+ for(BlockInfo b : file.getBlocks()) {
assertBlockCollection(blkManager, file, b);
}
return file;
}
static void assertBlockCollection(final BlockManager blkManager,
- final INodeFile file, final BlockInfoContiguous b) {
+ final INodeFile file, final BlockInfo b) {
Assert.assertSame(b, blkManager.getStoredBlock(b));
Assert.assertSame(file, blkManager.getBlockCollection(b));
Assert.assertSame(file, b.getBlockCollection());
@@ -146,10 +146,10 @@ public class TestSnapshotBlocksMap {
{
final INodeFile f2 = assertBlockCollection(file2.toString(), 3, fsdir,
blockmanager);
- BlockInfoContiguous[] blocks = f2.getBlocks();
+ BlockInfo[] blocks = f2.getBlocks();
hdfs.delete(sub2, true);
// The INode should have been removed from the blocksMap
- for(BlockInfoContiguous b : blocks) {
+ for(BlockInfo b : blocks) {
assertNull(blockmanager.getBlockCollection(b));
}
}
@@ -177,7 +177,7 @@ public class TestSnapshotBlocksMap {
// Check the block information for file0
final INodeFile f0 = assertBlockCollection(file0.toString(), 4, fsdir,
blockmanager);
- BlockInfoContiguous[] blocks0 = f0.getBlocks();
+ BlockInfo[] blocks0 = f0.getBlocks();
// Also check the block information for snapshot of file0
Path snapshotFile0 = SnapshotTestHelper.getSnapshotPath(sub1, "s0",
@@ -187,7 +187,7 @@ public class TestSnapshotBlocksMap {
// Delete file0
hdfs.delete(file0, true);
// Make sure the blocks of file0 is still in blocksMap
- for(BlockInfoContiguous b : blocks0) {
+ for(BlockInfo b : blocks0) {
assertNotNull(blockmanager.getBlockCollection(b));
}
assertBlockCollection(snapshotFile0.toString(), 4, fsdir, blockmanager);
@@ -201,7 +201,7 @@ public class TestSnapshotBlocksMap {
hdfs.deleteSnapshot(sub1, "s1");
// Make sure the first block of file0 is still in blocksMap
- for(BlockInfoContiguous b : blocks0) {
+ for(BlockInfo b : blocks0) {
assertNotNull(blockmanager.getBlockCollection(b));
}
assertBlockCollection(snapshotFile0.toString(), 4, fsdir, blockmanager);
@@ -293,7 +293,7 @@ public class TestSnapshotBlocksMap {
hdfs.append(bar);
INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
- BlockInfoContiguous[] blks = barNode.getBlocks();
+ BlockInfo[] blks = barNode.getBlocks();
assertEquals(1, blks.length);
assertEquals(BLOCKSIZE, blks[0].getNumBytes());
ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
@@ -331,7 +331,7 @@ public class TestSnapshotBlocksMap {
hdfs.append(bar);
INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
- BlockInfoContiguous[] blks = barNode.getBlocks();
+ BlockInfo[] blks = barNode.getBlocks();
assertEquals(1, blks.length);
ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
cluster.getNameNodeRpc()
@@ -370,7 +370,7 @@ public class TestSnapshotBlocksMap {
hdfs.append(bar);
INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
- BlockInfoContiguous[] blks = barNode.getBlocks();
+ BlockInfo[] blks = barNode.getBlocks();
assertEquals(1, blks.length);
ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
cluster.getNameNodeRpc()
@@ -421,7 +421,7 @@ public class TestSnapshotBlocksMap {
out.write(testData);
out.close();
INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
- BlockInfoContiguous[] blks = barNode.getBlocks();
+ BlockInfo[] blks = barNode.getBlocks();
assertEquals(1, blks.length);
assertEquals(testData.length, blks[0].getNumBytes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index cdd655e..139a37e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -262,12 +262,12 @@ public class TestSnapshotDeletion {
DFSTestUtil.createFile(hdfs, tempFile, BLOCKSIZE, REPLICATION, seed);
final INodeFile temp = TestSnapshotBlocksMap.assertBlockCollection(
tempFile.toString(), 1, fsdir, blockmanager);
- BlockInfoContiguous[] blocks = temp.getBlocks();
+ BlockInfo[] blocks = temp.getBlocks();
hdfs.delete(tempDir, true);
// check dir's quota usage
checkQuotaUsageComputation(dir, 8, BLOCKSIZE * REPLICATION * 3);
// check blocks of tempFile
- for (BlockInfoContiguous b : blocks) {
+ for (BlockInfo b : blocks) {
assertNull(blockmanager.getBlockCollection(b));
}
@@ -344,7 +344,7 @@ public class TestSnapshotDeletion {
// while deletion, we add diff for subsub and metaChangeFile1, and remove
// newFile
checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 4);
- for (BlockInfoContiguous b : blocks) {
+ for (BlockInfo b : blocks) {
assertNull(blockmanager.getBlockCollection(b));
}
@@ -481,7 +481,7 @@ public class TestSnapshotDeletion {
final INodeFile toDeleteFileNode = TestSnapshotBlocksMap
.assertBlockCollection(toDeleteFile.toString(), 1, fsdir, blockmanager);
- BlockInfoContiguous[] blocks = toDeleteFileNode.getBlocks();
+ BlockInfo[] blocks = toDeleteFileNode.getBlocks();
// create snapshot s0 on dir
SnapshotTestHelper.createSnapshot(hdfs, dir, "s0");
@@ -507,7 +507,7 @@ public class TestSnapshotDeletion {
// metaChangeDir's diff, dir's diff. diskspace: remove toDeleteFile, and
// metaChangeFile's replication factor decreases
checkQuotaUsageComputation(dir, 6, 2 * BLOCKSIZE * REPLICATION - BLOCKSIZE);
- for (BlockInfoContiguous b : blocks) {
+ for (BlockInfo b : blocks) {
assertNull(blockmanager.getBlockCollection(b));
}
@@ -801,7 +801,7 @@ public class TestSnapshotDeletion {
FileStatus statusBeforeDeletion13 = hdfs.getFileStatus(file13_s1);
INodeFile file14Node = TestSnapshotBlocksMap.assertBlockCollection(
file14_s2.toString(), 1, fsdir, blockmanager);
- BlockInfoContiguous[] blocks_14 = file14Node.getBlocks();
+ BlockInfo[] blocks_14 = file14Node.getBlocks();
TestSnapshotBlocksMap.assertBlockCollection(file15_s2.toString(), 1, fsdir,
blockmanager);
@@ -838,7 +838,7 @@ public class TestSnapshotDeletion {
modDirStr + "file15");
assertFalse(hdfs.exists(file14_s1));
assertFalse(hdfs.exists(file15_s1));
- for (BlockInfoContiguous b : blocks_14) {
+ for (BlockInfo b : blocks_14) {
assertNull(blockmanager.getBlockCollection(b));
}
[40/50] [abbrv] hadoop git commit: YARN-1462. Made RM write
application tags to timeline server and exposed them to users via generic
history web UI and REST API. Contributed by Xuan Gong.
Posted by zj...@apache.org.
YARN-1462. Made RM write application tags to timeline server and exposed them to users via generic history web UI and REST API. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2101ca1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2101ca1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2101ca1
Branch: refs/heads/YARN-2928
Commit: e2101ca195ab003f53d12f58d5b7604b1dc27fc9
Parents: af69d9c
Author: Zhijie Shen <zj...@apache.org>
Authored: Sat May 30 09:35:59 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:59 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/mapred/NotRunningJob.java | 2 +-
.../mapred/TestClientServiceDelegate.java | 4 +--
.../apache/hadoop/mapred/TestYARNRunner.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 ++
.../yarn/api/records/ApplicationReport.java | 4 ++-
.../hadoop/yarn/client/ProtocolHATestBase.java | 2 +-
.../yarn/client/api/impl/TestAHSClient.java | 8 ++---
.../yarn/client/api/impl/TestYarnClient.java | 9 ++---
.../hadoop/yarn/client/cli/TestYarnCLI.java | 27 ++++++++------
.../hadoop/yarn/api/TestApplicatonReport.java | 3 +-
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 19 ++++++++--
...pplicationHistoryManagerOnTimelineStore.java | 9 +++++
.../metrics/ApplicationMetricsConstants.java | 1 +
.../metrics/ApplicationCreatedEvent.java | 10 +++++-
.../metrics/SystemMetricsPublisher.java | 4 ++-
.../applicationsmanager/MockAsm.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 37 ++++++++++++++++++++
.../resourcemanager/webapp/TestRMWebApp.java | 2 +-
.../src/site/markdown/TimelineServer.md | 30 ++++++++++++++--
20 files changed, 144 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
index 03552e4..1b26cd3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
@@ -90,7 +90,7 @@ public class NotRunningJob implements MRClientProtocol {
return ApplicationReport.newInstance(unknownAppId, unknownAttemptId,
"N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
"N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
- YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null);
}
NotRunningJob(ApplicationReport applicationReport, JobState jobState) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
index b85f18d..c1c03df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
@@ -516,7 +516,7 @@ public class TestClientServiceDelegate {
return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
"appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
- "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null);
}
private ApplicationReport getRunningApplicationReport(String host, int port) {
@@ -526,7 +526,7 @@ public class TestClientServiceDelegate {
return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
"appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics",
"url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
- YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null);
}
private ResourceMgrDelegate getRMDelegate() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 0e53ab0..380fd62 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -196,7 +196,7 @@ public class TestYARNRunner extends TestCase {
ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp",
"tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp",
0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f,
- "tmp", null));
+ "tmp", null, null));
yarnRunner.killJob(jobId);
verify(clientDelegate).killJob(jobId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2c48701..6f5faa9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -584,6 +584,9 @@ Release 2.7.1 - UNRELEASED
YARN-3723. Need to clearly document primaryFilter and otherInfo value type.
(Zhijie Shen via xgong)
+ YARN-1462. Made RM write application tags to timeline server and exposed them
+ to users via generic history web UI and REST API. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
index e5d7254..99e4e2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
@@ -59,7 +59,8 @@ public abstract class ApplicationReport {
YarnApplicationState state, String diagnostics, String url,
long startTime, long finishTime, FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
- float progress, String applicationType, Token amRmToken) {
+ float progress, String applicationType, Token amRmToken,
+ Set<String> tags) {
ApplicationReport report = Records.newRecord(ApplicationReport.class);
report.setApplicationId(applicationId);
report.setCurrentApplicationAttemptId(applicationAttemptId);
@@ -80,6 +81,7 @@ public abstract class ApplicationReport {
report.setProgress(progress);
report.setApplicationType(applicationType);
report.setAMRMToken(amRmToken);
+ report.setApplicationTags(tags);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index 903dd94..2260de5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -647,7 +647,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
"fakeQueue", "fakeApplicationName", "localhost", 0, null,
YarnApplicationState.FINISHED, "fake an application report", "",
1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f,
- "fakeApplicationType", null);
+ "fakeApplicationType", null, null);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
index c3e3c41..12d0602 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
@@ -334,7 +334,7 @@ public class TestAHSClient {
"queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
- null);
+ null, null);
List<ApplicationReport> applicationReports =
new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
@@ -391,7 +391,7 @@ public class TestAHSClient {
"queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f,
- "NON-YARN", null);
+ "NON-YARN", null, null);
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -401,7 +401,7 @@ public class TestAHSClient {
"queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f,
- "MAPREDUCE", null);
+ "MAPREDUCE", null, null);
applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -411,7 +411,7 @@ public class TestAHSClient {
"queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
- "NON-MAPREDUCE", null);
+ "NON-MAPREDUCE", null, null);
applicationReports.add(newApplicationReport4);
reports = applicationReports;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index bc40b9a..b7608b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -613,7 +613,8 @@ public class TestYarnClient {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
List<ApplicationAttemptReport> appAttempts = new ArrayList<ApplicationAttemptReport>();
@@ -689,7 +690,7 @@ public class TestYarnClient {
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
- null);
+ null, null);
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -698,7 +699,7 @@ public class TestYarnClient {
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
- null);
+ null, null);
applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -709,7 +710,7 @@ public class TestYarnClient {
"user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
- "NON-MAPREDUCE", null);
+ "NON-MAPREDUCE", null, null);
applicationReports.add(newApplicationReport4);
return applicationReports;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 1013958..70c7d82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -104,7 +104,7 @@ public class TestYarnCLI {
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
- null);
+ null, null);
newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
@@ -350,7 +350,8 @@ public class TestYarnCLI {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
@@ -360,7 +361,7 @@ public class TestYarnCLI {
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
- null);
+ null, null);
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -369,7 +370,7 @@ public class TestYarnCLI {
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
- null);
+ null, null);
applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -378,7 +379,7 @@ public class TestYarnCLI {
"user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
- null);
+ null, null);
applicationReports.add(newApplicationReport4);
ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
@@ -387,7 +388,7 @@ public class TestYarnCLI {
"user5", "queue5", "appname5", "host5", 128, null,
YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5,
FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE",
- null);
+ null, null);
applicationReports.add(newApplicationReport5);
ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
@@ -396,7 +397,7 @@ public class TestYarnCLI {
"user6", "queue6", "appname6", "host6", 129, null,
YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6,
FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG",
- null);
+ null, null);
applicationReports.add(newApplicationReport6);
// Test command yarn application -list
@@ -820,7 +821,8 @@ public class TestYarnCLI {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport2);
int result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@@ -833,7 +835,8 @@ public class TestYarnCLI {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@@ -868,7 +871,8 @@ public class TestYarnCLI {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport2);
int result = cli.run(new String[] { "application", "-movetoqueue",
@@ -883,7 +887,8 @@ public class TestYarnCLI {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
result = cli.run(new String[] { "application", "-movetoqueue",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
index 9302d4b..a2ea0a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
@@ -58,7 +58,8 @@ public class TestApplicatonReport {
ApplicationReport.newInstance(appId, appAttemptId, "user", "queue",
"appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
- "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
+ null);
return appReport;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index 479858f..766e0cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -139,7 +139,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(),
appHistory.getFinalApplicationStatus(), null, "", 100,
- appHistory.getApplicationType(), null);
+ appHistory.getApplicationType(), null, null);
}
private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 9bfd2d6..0c7fdc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -19,11 +19,14 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -249,6 +252,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
YarnApplicationState state = YarnApplicationState.ACCEPTED;
ApplicationResourceUsageReport appResources = null;
+ Set<String> appTags = null;
Map<ApplicationAccessType, String> appViewACLs =
new HashMap<ApplicationAccessType, String>();
Map<String, Object> entityInfo = entity.getOtherInfo();
@@ -270,7 +274,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
- null, progress, type, null), appViewACLs);
+ null, progress, type, null, appTags), appViewACLs);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue =
@@ -295,6 +299,17 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
appResources=ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds);
}
+ if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
+ appTags = new HashSet<String>();
+ Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
+ if (obj != null && obj instanceof Collection<?>) {
+ for(Object o : (Collection<?>)obj) {
+ if (o != null) {
+ appTags.add(o.toString());
+ }
+ }
+ }
+ }
}
List<TimelineEvent> events = entity.getEvents();
if (events != null) {
@@ -347,7 +362,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
- null, progress, type, null), appViewACLs);
+ null, progress, type, null, appTags), appViewACLs);
}
private static ApplicationAttemptReport convertToApplicationAttemptReport(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 8672953..dbd75ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -22,7 +22,9 @@ import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
@@ -185,6 +187,9 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals(Integer.MAX_VALUE + 3L
+ +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
+ Assert.assertEquals(2, app.getApplicationTags().size());
+ Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1"));
+ Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
if ((i == 1 && callerUGI != null &&
@@ -471,6 +476,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
"user2");
}
+ Set<String> appTags = new HashSet<String>();
+ appTags.add("Test_APP_TAGS_1");
+ appTags.add("Test_APP_TAGS_2");
+ entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, appTags);
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index df8eecb..f452410 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -73,4 +73,5 @@ public class ApplicationMetricsConstants {
public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
"YARN_APPLICATION_LATEST_APP_ATTEMPT";
+ public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
index 2373b3b..7c43aa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+import java.util.Set;
+
import org.apache.hadoop.yarn.api.records.ApplicationId;
public class ApplicationCreatedEvent extends
@@ -29,6 +31,7 @@ public class ApplicationCreatedEvent extends
private String user;
private String queue;
private long submittedTime;
+ private Set<String> appTags;
public ApplicationCreatedEvent(ApplicationId appId,
String name,
@@ -36,7 +39,8 @@ public class ApplicationCreatedEvent extends
String user,
String queue,
long submittedTime,
- long createdTime) {
+ long createdTime,
+ Set<String> appTags) {
super(SystemMetricsEventType.APP_CREATED, createdTime);
this.appId = appId;
this.name = name;
@@ -44,6 +48,7 @@ public class ApplicationCreatedEvent extends
this.user = user;
this.queue = queue;
this.submittedTime = submittedTime;
+ this.appTags = appTags;
}
@Override
@@ -75,4 +80,7 @@ public class ApplicationCreatedEvent extends
return submittedTime;
}
+ public Set<String> getAppTags() {
+ return appTags;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 2828aec..63461b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -107,7 +107,7 @@ public class SystemMetricsPublisher extends CompositeService {
app.getUser(),
app.getQueue(),
app.getSubmitTime(),
- createdTime));
+ createdTime, app.getApplicationTags()));
}
}
@@ -252,6 +252,8 @@ public class SystemMetricsPublisher extends CompositeService {
event.getQueue());
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
event.getSubmittedTime());
+ entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
+ event.getAppTags());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index e52b054..34247e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -313,7 +313,7 @@ public abstract class MockAsm extends MockApps {
getName(), null, 0, null, null, getDiagnostics().toString(),
getTrackingUrl(), getStartTime(), getFinishTime(),
getFinalApplicationStatus(), usageReport , null, getProgress(),
- type, null);
+ type, null, null);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 7c4b5e9..b122bc4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -21,7 +21,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.util.Collection;
import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -143,6 +147,8 @@ public class TestSystemMetricsPublisher {
Assert.assertEquals(app.getSubmitTime(),
entity.getOtherInfo().get(
ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO));
+ Assert.assertTrue(verifyAppTags(app.getApplicationTags(),
+ entity.getOtherInfo()));
if (i == 1) {
Assert.assertEquals("uers1,user2",
entity.getOtherInfo().get(
@@ -352,6 +358,10 @@ public class TestSystemMetricsPublisher {
FinalApplicationStatus.UNDEFINED);
when(app.getRMAppMetrics()).thenReturn(
new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
+ Set<String> appTags = new HashSet<String>();
+ appTags.add("test");
+ appTags.add("tags");
+ when(app.getApplicationTags()).thenReturn(appTags);
return app;
}
@@ -392,4 +402,31 @@ public class TestSystemMetricsPublisher {
return container;
}
+ private static boolean verifyAppTags(Set<String> appTags,
+ Map<String, Object> entityInfo) {
+ if (!entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
+ return false;
+ }
+ Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
+ if (obj instanceof Collection<?>) {
+ Collection<?> collection = (Collection<?>) obj;
+ if (collection.size() != appTags.size()) {
+ return false;
+ }
+ for (String appTag : appTags) {
+ boolean match = false;
+ for (Object o : collection) {
+ if (o.toString().equals(appTag)) {
+ match = true;
+ break;
+ }
+ }
+ if (!match) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index 0290421..38eebfe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -253,7 +253,7 @@ public class TestRMWebApp {
app.getStartTime(), app.getFinishTime(),
app.getFinalApplicationStatus(),
(ApplicationResourceUsageReport) null, app.getTrackingUrl(),
- app.getProgress(), app.getApplicationType(), (Token) null);
+ app.getProgress(), app.getApplicationType(), (Token) null, null);
appReports.add(appReport);
}
GetApplicationsResponse response = mock(GetApplicationsResponse.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2101ca1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 90f9512..1b66606 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -1125,7 +1125,27 @@ Response Body:
"submittedTime":1430424769395,
"startedTime":1430424769395,
"finishedTime":1430424776594,
- "elapsedTime":7199
+ "elapsedTime":7199},
+ {
+ "appId":"application_1430424020775_0001",
+ "currentAppAttemptId":"appattempt_1430424020775_0001_000001",
+ "user":"zshen",
+ "name":"QuasiMonteCarlo",
+ "queue":"default",
+ "type":"MAPREDUCE",
+ "host":"localhost",
+ "rpcPort":56264,
+ "appState":"FINISHED",
+ "progress":100.0,
+ "diagnosticsInfo":"",
+ "originalTrackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:19888/jobhistory/job/job_1430424020775_0001",
+ "trackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:8088/proxy/application_1430424020775_0001/",
+ "finalAppStatus":"SUCCEEDED",
+ "submittedTime":1430424053809,
+ "startedTime":1430424072153,
+ "finishedTime":1430424776594,
+ "elapsedTime":18344,
+ "applicationTags":"mrapplication,ta-example"
}
]
}
@@ -1227,6 +1247,7 @@ Response Body:
<startedTime>1430424053809</startedTime>
<finishedTime>1430424072153</finishedTime>
<elapsedTime>18344</elapsedTime>
+ <applicationTags>mrapplication,ta-example</applicationTags>
</app>
</apps>
@@ -1275,7 +1296,8 @@ None
| `allocatedVCores` | int | The sum of virtual cores allocated to the application's running containers |
| `currentAppAttemptId` | string | The latest application attempt ID |
| `host` | string | The host of the ApplicationMaster |
-| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared. |
+| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared |
+| `applicationTags` | string | The application tags. |
### Response Examples:
@@ -1311,7 +1333,8 @@ Response Body:
"submittedTime": 1430424053809,
"startedTime": 1430424053809,
"finishedTime": 1430424072153,
- "elapsedTime": 18344
+ "elapsedTime": 18344,
+ "applicationTags": mrapplication,tag-example
}
#### XML response
@@ -1349,6 +1372,7 @@ Response Body:
<startedTime>1430424053809</startedTime>
<finishedTime>1430424072153</finishedTime>
<elapsedTime>18344</elapsedTime>
+ <applicationTags>mrapplication,ta-example</applicationTags>
</app>
## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List
[33/50] [abbrv] hadoop git commit: HDFS-7609. Avoid retry cache
collision when Standby NameNode loading edits. Contributed by Ming Ma.
Posted by zj...@apache.org.
HDFS-7609. Avoid retry cache collision when Standby NameNode loading edits. Contributed by Ming Ma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58b91c94
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58b91c94
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58b91c94
Branch: refs/heads/YARN-2928
Commit: 58b91c94086f7e2cacd07087c9e7a0d28b2355d3
Parents: ab02e31
Author: Jing Zhao <ji...@apache.org>
Authored: Fri May 29 11:05:13 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/namenode/FSNamesystem.java | 20 -----------
.../hdfs/server/namenode/NameNodeRpcServer.java | 22 ++++++++++--
.../namenode/ha/TestRetryCacheWithHA.java | 37 ++++++++++++++++++--
4 files changed, 58 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b91c94/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 23699a3..cc8235c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -822,6 +822,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7401. Add block info to DFSInputStream' WARN message when it adds
node to deadNodes (Arshad Mohammad via vinayakumarb)
+ HDFS-7609. Avoid retry cache collision when Standby NameNode loading edits.
+ (Ming Ma via jing9)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b91c94/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 798f8d5..5ed069d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1887,7 +1887,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void concat(String target, String [] srcs, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
waitForLoadingFSImage();
HdfsFileStatus stat = null;
boolean success = false;
@@ -2162,7 +2161,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
waitForLoadingFSImage();
HdfsFileStatus auditStat = null;
- checkOperation(OperationCategory.WRITE);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2379,7 +2377,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new InvalidPathException(src);
}
blockManager.verifyReplication(src, replication, clientMachine);
- checkOperation(OperationCategory.WRITE);
if (blockSize < minBlockSize) {
throw new IOException("Specified block size is less than configured" +
" minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY
@@ -2786,7 +2783,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
LocatedBlock lb = null;
HdfsFileStatus stat = null;
FSPermissionChecker pc = getPermissionChecker();
- checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
writeLock();
try {
@@ -3081,7 +3077,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean renameTo(String src, String dst, boolean logRetryCache)
throws IOException {
waitForLoadingFSImage();
- checkOperation(OperationCategory.WRITE);
FSDirRenameOp.RenameOldResult ret = null;
writeLock();
try {
@@ -3107,7 +3102,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean logRetryCache, Options.Rename... options)
throws IOException {
waitForLoadingFSImage();
- checkOperation(OperationCategory.WRITE);
Map.Entry<BlocksMapUpdateInfo, HdfsFileStatus> res = null;
writeLock();
try {
@@ -3144,7 +3138,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean delete(String src, boolean recursive, boolean logRetryCache)
throws IOException {
waitForLoadingFSImage();
- checkOperation(OperationCategory.WRITE);
BlocksMapUpdateInfo toRemovedBlocks = null;
writeLock();
boolean ret = false;
@@ -5762,8 +5755,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
String clientName, ExtendedBlock oldBlock, ExtendedBlock newBlock,
DatanodeID[] newNodes, String[] newStorageIDs, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
-
LOG.info("updatePipeline(" + oldBlock.getLocalBlock()
+ ", newGS=" + newBlock.getGenerationStamp()
+ ", newLength=" + newBlock.getNumBytes()
@@ -6744,7 +6735,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void renameSnapshot(
String path, String snapshotOldName, String snapshotNewName,
boolean logRetryCache) throws IOException {
- checkOperation(OperationCategory.WRITE);
boolean success = false;
writeLock();
try {
@@ -6832,7 +6822,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
void deleteSnapshot(String snapshotRoot, String snapshotName,
boolean logRetryCache) throws IOException {
- checkOperation(OperationCategory.WRITE);
boolean success = false;
writeLock();
BlocksMapUpdateInfo blocksToBeDeleted = null;
@@ -7057,7 +7046,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
long addCacheDirective(CacheDirectiveInfo directive,
EnumSet<CacheFlag> flags, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
CacheDirectiveInfo effectiveDirective = null;
if (!flags.contains(CacheFlag.FORCE)) {
cacheManager.waitForRescanIfNeeded();
@@ -7085,7 +7073,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void modifyCacheDirective(CacheDirectiveInfo directive,
EnumSet<CacheFlag> flags, boolean logRetryCache) throws IOException {
- checkOperation(OperationCategory.WRITE);
boolean success = false;
if (!flags.contains(CacheFlag.FORCE)) {
cacheManager.waitForRescanIfNeeded();
@@ -7109,7 +7096,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
void removeCacheDirective(long id, boolean logRetryCache) throws IOException {
- checkOperation(OperationCategory.WRITE);
boolean success = false;
writeLock();
try {
@@ -7148,7 +7134,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void addCachePool(CachePoolInfo req, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
writeLock();
boolean success = false;
String poolInfoStr = null;
@@ -7170,7 +7155,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void modifyCachePool(CachePoolInfo req, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
writeLock();
boolean success = false;
try {
@@ -7192,7 +7176,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void removeCachePool(String cachePoolName, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
writeLock();
boolean success = false;
try {
@@ -7387,7 +7370,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
String src = srcArg;
HdfsFileStatus resultingStat = null;
checkSuperuserPrivilege();
- checkOperation(OperationCategory.WRITE);
final byte[][] pathComponents =
FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
@@ -7473,7 +7455,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
HdfsFileStatus auditStat = null;
writeLock();
try {
@@ -7521,7 +7502,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void removeXAttr(String src, XAttr xAttr, boolean logRetryCache)
throws IOException {
- checkOperation(OperationCategory.WRITE);
HdfsFileStatus auditStat = null;
writeLock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b91c94/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 0d416a6..dafa23e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -616,7 +616,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
throw new IOException("create: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
-
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache, null);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return (HdfsFileStatus) cacheEntry.getPayload();
@@ -647,6 +647,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
stateChangeLog.debug("*DIR* NameNode.append: file "
+src+" for "+clientName+" at "+clientMachine);
}
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
null);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -794,6 +795,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs)
throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -838,7 +840,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
throw new IOException("rename: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
-
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return true; // Return previous response
@@ -859,6 +861,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void concat(String trg, String[] src) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -884,6 +887,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
throw new IOException("rename: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -922,6 +926,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
stateChangeLog.debug("*DIR* Namenode.delete: src=" + src
+ ", recursive=" + recursive);
}
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return true; // Return previous response
@@ -1207,6 +1212,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void createSymlink(String target, String link, FsPermission dirPerms,
boolean createParent) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -1537,6 +1543,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
throw new IOException("createSnapshot: Pathname too long. Limit "
+ MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
}
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion(retryCache,
null);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1558,6 +1565,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void deleteSnapshot(String snapshotRoot, String snapshotName)
throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
metrics.incrDeleteSnapshotOps();
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1596,6 +1604,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
if (snapshotNewName == null || snapshotNewName.isEmpty()) {
throw new IOException("The new snapshot name is null or empty.");
}
+ namesystem.checkOperation(OperationCategory.WRITE);
metrics.incrRenameSnapshotOps();
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1635,6 +1644,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public long addCacheDirective(
CacheDirectiveInfo path, EnumSet<CacheFlag> flags) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntryWithPayload cacheEntry = RetryCache.waitForCompletion
(retryCache, null);
if (cacheEntry != null && cacheEntry.isSuccess()) {
@@ -1656,6 +1666,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void modifyCacheDirective(
CacheDirectiveInfo directive, EnumSet<CacheFlag> flags) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return;
@@ -1673,6 +1684,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void removeCacheDirective(long id) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return;
@@ -1699,6 +1711,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override //ClientProtocol
public void addCachePool(CachePoolInfo info) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -1715,6 +1728,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void modifyCachePool(CachePoolInfo info) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -1731,6 +1745,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void removeCachePool(String cachePoolName) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return;
@@ -1793,6 +1808,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void createEncryptionZone(String src, String keyName)
throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return;
@@ -1824,6 +1840,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
@@ -1853,6 +1870,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
@Override // ClientProtocol
public void removeXAttr(String src, XAttr xAttr) throws IOException {
checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
if (cacheEntry != null && cacheEntry.isSuccess()) {
return; // Return previous response
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b91c94/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
index e3572ab..d202fb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
@@ -215,7 +215,8 @@ public class TestRetryCacheWithHA {
abstract class AtMostOnceOp {
private final String name;
final DFSClient client;
-
+ int expectedUpdateCount = 0;
+
AtMostOnceOp(String name, DFSClient client) {
this.name = name;
this.client = client;
@@ -225,6 +226,9 @@ public class TestRetryCacheWithHA {
abstract void invoke() throws Exception;
abstract boolean checkNamenodeBeforeReturn() throws Exception;
abstract Object getResult();
+ int getExpectedCacheUpdateCount() {
+ return expectedUpdateCount;
+ }
}
/** createSnapshot operaiton */
@@ -604,7 +608,7 @@ public class TestRetryCacheWithHA {
class DeleteOp extends AtMostOnceOp {
private final String target;
private boolean deleted;
-
+
DeleteOp(DFSClient client, String target) {
super("delete", client);
this.target = target;
@@ -614,12 +618,14 @@ public class TestRetryCacheWithHA {
void prepare() throws Exception {
Path p = new Path(target);
if (!dfs.exists(p)) {
+ expectedUpdateCount++;
DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
}
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
deleted = client.delete(target, true);
}
@@ -655,12 +661,14 @@ public class TestRetryCacheWithHA {
void prepare() throws Exception {
Path p = new Path(target);
if (!dfs.exists(p)) {
+ expectedUpdateCount++;
DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
}
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.createSymlink(target, link, false);
}
@@ -773,11 +781,13 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
dfs.addCachePool(new CachePoolInfo(directive.getPool()));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
result = client.addCacheDirective(directive, EnumSet.of(CacheFlag.FORCE));
}
@@ -819,12 +829,15 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
dfs.addCachePool(new CachePoolInfo(directive.getPool()));
+ expectedUpdateCount++;
id = client.addCacheDirective(directive, EnumSet.of(CacheFlag.FORCE));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.modifyCacheDirective(
new CacheDirectiveInfo.Builder().
setId(id).
@@ -875,12 +888,15 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
dfs.addCachePool(new CachePoolInfo(directive.getPool()));
+ expectedUpdateCount++;
id = dfs.addCacheDirective(directive, EnumSet.of(CacheFlag.FORCE));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.removeCacheDirective(id);
}
@@ -922,6 +938,7 @@ public class TestRetryCacheWithHA {
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.addCachePool(new CachePoolInfo(pool));
}
@@ -954,11 +971,13 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
client.addCachePool(new CachePoolInfo(pool).setLimit(10l));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.modifyCachePool(new CachePoolInfo(pool).setLimit(99l));
}
@@ -991,11 +1010,13 @@ public class TestRetryCacheWithHA {
@Override
void prepare() throws Exception {
+ expectedUpdateCount++;
client.addCachePool(new CachePoolInfo(pool));
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.removeCachePool(pool);
}
@@ -1030,12 +1051,14 @@ public class TestRetryCacheWithHA {
void prepare() throws Exception {
Path p = new Path(src);
if (!dfs.exists(p)) {
+ expectedUpdateCount++;
DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
}
}
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.setXAttr(src, "user.key", "value".getBytes(),
EnumSet.of(XAttrSetFlag.CREATE));
}
@@ -1072,7 +1095,9 @@ public class TestRetryCacheWithHA {
void prepare() throws Exception {
Path p = new Path(src);
if (!dfs.exists(p)) {
+ expectedUpdateCount++;
DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0);
+ expectedUpdateCount++;
client.setXAttr(src, "user.key", "value".getBytes(),
EnumSet.of(XAttrSetFlag.CREATE));
}
@@ -1080,6 +1105,7 @@ public class TestRetryCacheWithHA {
@Override
void invoke() throws Exception {
+ expectedUpdateCount++;
client.removeXAttr(src, "user.key");
}
@@ -1316,6 +1342,13 @@ public class TestRetryCacheWithHA {
assertTrue("CacheUpdated on NN0: " + updatedNN0, updatedNN0 > 0);
// Cache updated metrics on NN0 should be >0 since NN1 applied the editlog
assertTrue("CacheUpdated on NN1: " + updatedNN1, updatedNN1 > 0);
+ long expectedUpdateCount = op.getExpectedCacheUpdateCount();
+ if (expectedUpdateCount > 0) {
+ assertEquals("CacheUpdated on NN0: " + updatedNN0, expectedUpdateCount,
+ updatedNN0);
+ assertEquals("CacheUpdated on NN0: " + updatedNN1, expectedUpdateCount,
+ updatedNN1);
+ }
}
/**
[25/50] [abbrv] hadoop git commit: HADOOP-11930. test-patch in
offline mode should tell maven to be in offline mode (Sean Busbey via aw)
Posted by zj...@apache.org.
HADOOP-11930. test-patch in offline mode should tell maven to be in offline mode (Sean Busbey via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b4aeba0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b4aeba0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b4aeba0
Branch: refs/heads/YARN-2928
Commit: 4b4aeba0db26dae2d6449b33714fcb5fc5bb7efd
Parents: 81f0adb
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 12:11:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 35 ++++++++++++--------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
2 files changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4aeba0/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index f760951..1409467 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -33,6 +33,8 @@ function setup_defaults
else
MVN=${MAVEN_HOME}/bin/mvn
fi
+ # This parameter needs to be kept as an array
+ MAVEN_ARGS=()
PROJECT_NAME=hadoop
HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
@@ -756,6 +758,11 @@ function parse_args
esac
done
+ # if we requested offline, pass that to mvn
+ if [[ ${OFFLINE} == "true" ]] ; then
+ MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline)
+ fi
+
# we need absolute dir for ${BASEDIR}
cd "${CWD}"
BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P)
@@ -1027,7 +1034,7 @@ function precheck_without_patch
if [[ $? == 1 ]]; then
echo "Compiling ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
+ echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
if [[ $? != 0 ]] ; then
echo "${PATCH_BRANCH} compilation is broken?"
add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken."
@@ -1041,7 +1048,7 @@ function precheck_without_patch
if [[ $? == 1 ]]; then
echo "Javadoc'ing ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
+ echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
if [[ $? != 0 ]] ; then
echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?"
add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken."
@@ -1055,7 +1062,7 @@ function precheck_without_patch
if [[ $? == 1 ]]; then
echo "site creation for ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
+ echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
if [[ $? != 0 ]] ; then
echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?"
add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken."
@@ -1633,12 +1640,12 @@ function check_javadoc
start_clock
if [[ -d hadoop-project ]]; then
- (cd hadoop-project; "${MVN}" install > /dev/null 2>&1)
+ (cd hadoop-project; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
fi
if [[ -d hadoop-common-project/hadoop-annotations ]]; then
- (cd hadoop-common-project/hadoop-annotations; "${MVN}" install > /dev/null 2>&1)
+ (cd hadoop-common-project/hadoop-annotations; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
fi
- echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt" "${MVN}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
+ echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt"
numBranchJavadocWarnings=$?
count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt"
@@ -1688,7 +1695,7 @@ function check_site
start_clock
echo "site creation for ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
+ echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
if [[ $? != 0 ]] ; then
echo "Site compilation is broken"
add_jira_table -1 site "Site compilation is broken."
@@ -1734,7 +1741,7 @@ function check_javac
start_clock
- echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch
+ echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch
if [[ $? != 0 ]] ; then
add_jira_table -1 javac "The patch appears to cause the build to fail."
return 2
@@ -1784,7 +1791,7 @@ function check_apachelicense
start_clock
- echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" apache-rat:check -D${PROJECT_NAME}PatchProcess
+ echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" apache-rat:check -D${PROJECT_NAME}PatchProcess
#shellcheck disable=SC2038
find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt"
@@ -1839,7 +1846,7 @@ function check_mvn_install
big_console_header "Installing all of the jars"
start_clock
- echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess
+ echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" "${MAVEN_ARGS[@]}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess
retval=$?
if [[ ${retval} != 0 ]]; then
add_jira_table -1 install "The patch causes mvn install to fail."
@@ -1877,7 +1884,7 @@ function findbugs_mvnrunner
local logfile=$2
local warnings_file=$3
- echo_and_redirect "${logfile}" "${MVN}" clean test findbugs:findbugs -DskipTests \
+ echo_and_redirect "${logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean test findbugs:findbugs -DskipTests \
"-D${PROJECT_NAME}PatchProcess" < /dev/null
if [[ $? != 0 ]]; then
return 1
@@ -2120,7 +2127,7 @@ function check_mvn_eclipse
start_clock
- echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess
+ echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess
if [[ $? != 0 ]] ; then
add_jira_table -1 eclipse:eclipse "The patch failed to build with eclipse:eclipse."
return 1
@@ -2211,7 +2218,7 @@ function check_unittests
ordered_modules="${ordered_modules} ${hdfs_modules}"
if [[ ${building_common} -eq 0 ]]; then
echo " Building hadoop-common with -Pnative in order to provide libhadoop.so to the hadoop-hdfs unit tests."
- echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess"
+ echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" "${MAVEN_ARGS[@]}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess"
if [[ $? != 0 ]]; then
add_jira_table -1 "native" "Failed to build the native portion " \
"of hadoop-common prior to running the unit tests in ${ordered_modules}"
@@ -2231,7 +2238,7 @@ function check_unittests
test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt
echo " Running tests in ${module_suffix}"
- echo_and_redirect "${test_logfile}" "${MVN}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} -D${PROJECT_NAME}PatchProcess
+ echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} -D${PROJECT_NAME}PatchProcess
test_build_result=$?
add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4aeba0/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2b85c1c..6aa224c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -788,6 +788,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8429. Avoid stuck threads if there is an error in DomainSocketWatcher
that stops the thread. (zhouyingchao via cmccabe)
+ HADOOP-11930. test-patch in offline mode should tell maven to be in
+ offline mode (Sean Busbey via aw)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
[19/50] [abbrv] hadoop git commit: HADOOP-11894. Bump the version of
Apache HTrace to 3.2.0-incubating (Masatake Iwasaki via Colin P. McCabe)
Posted by zj...@apache.org.
HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating (Masatake Iwasaki via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81f0adb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81f0adb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81f0adb7
Branch: refs/heads/YARN-2928
Commit: 81f0adb7c4c8b2310e2bba02643b76544a0d1d9c
Parents: 3d5c0d4
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu May 28 12:00:55 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../apache/hadoop/tracing/SpanReceiverHost.java | 45 ++------------------
.../hadoop-common/src/site/markdown/Tracing.md | 12 +++---
.../java/org/apache/hadoop/hdfs/DFSClient.java | 16 ++-----
hadoop-project/pom.xml | 2 +-
5 files changed, 18 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 51eff78..2b85c1c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -620,6 +620,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12030. test-patch should only report on newly introduced
findbugs warnings. (Sean Busbey via aw)
+ HADOOP-11894. Bump the version of Apache HTrace to 3.2.0-incubating
+ (Masatake Iwasaki via Colin P. McCabe)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
index bf9479b..9a99e05 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/SpanReceiverHost.java
@@ -17,33 +17,23 @@
*/
package org.apache.hadoop.tracing;
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import java.util.UUID;
-import org.apache.commons.io.Charsets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.tracing.SpanReceiverInfo.ConfigurationPair;
-import org.apache.hadoop.tracing.TraceUtils;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.htrace.SpanReceiver;
import org.apache.htrace.SpanReceiverBuilder;
import org.apache.htrace.Trace;
+import org.apache.htrace.impl.LocalFileSpanReceiver;
/**
* This class provides functions for reading the names of SpanReceivers from
@@ -54,7 +44,7 @@ import org.apache.htrace.Trace;
@InterfaceAudience.Private
public class SpanReceiverHost implements TraceAdminProtocol {
public static final String SPAN_RECEIVERS_CONF_SUFFIX =
- "spanreceiver.classes";
+ "spanreceiver.classes";
private static final Log LOG = LogFactory.getLog(SpanReceiverHost.class);
private static final HashMap<String, SpanReceiverHost> hosts =
new HashMap<String, SpanReceiverHost>(1);
@@ -88,33 +78,6 @@ public class SpanReceiverHost implements TraceAdminProtocol {
private static List<ConfigurationPair> EMPTY = Collections.emptyList();
- private static String getUniqueLocalTraceFileName() {
- String tmp = System.getProperty("java.io.tmpdir", "/tmp");
- String nonce = null;
- BufferedReader reader = null;
- try {
- // On Linux we can get a unique local file name by reading the process id
- // out of /proc/self/stat. (There isn't any portable way to get the
- // process ID from Java.)
- reader = new BufferedReader(
- new InputStreamReader(new FileInputStream("/proc/self/stat"),
- Charsets.UTF_8));
- String line = reader.readLine();
- if (line == null) {
- throw new EOFException();
- }
- nonce = line.split(" ")[0];
- } catch (IOException e) {
- } finally {
- IOUtils.cleanup(LOG, reader);
- }
- if (nonce == null) {
- // If we can't use the process ID, use a random nonce.
- nonce = UUID.randomUUID().toString();
- }
- return new File(tmp, nonce).getAbsolutePath();
- }
-
private SpanReceiverHost(String confPrefix) {
this.confPrefix = confPrefix;
}
@@ -143,7 +106,7 @@ public class SpanReceiverHost implements TraceAdminProtocol {
// testing.
String pathKey = confPrefix + LOCAL_FILE_SPAN_RECEIVER_PATH_SUFFIX;
if (config.get(pathKey) == null) {
- String uniqueFile = getUniqueLocalTraceFileName();
+ String uniqueFile = LocalFileSpanReceiver.getUniqueLocalTraceFileName();
config.set(pathKey, uniqueFile);
if (LOG.isTraceEnabled()) {
LOG.trace("Set " + pathKey + " to " + uniqueFile);
@@ -191,7 +154,7 @@ public class SpanReceiverHost implements TraceAdminProtocol {
public synchronized SpanReceiverInfo[] listSpanReceivers()
throws IOException {
- SpanReceiverInfo info[] = new SpanReceiverInfo[receivers.size()];
+ SpanReceiverInfo[] info = new SpanReceiverInfo[receivers.size()];
int i = 0;
for(Map.Entry<Long, SpanReceiver> entry : receivers.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
index 76b5ee0..bf5685c 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
@@ -78,6 +78,10 @@ You also need to add the jar bundling SpanReceiver to the classpath of Hadoop
on each node. (LocalFileSpanReceiver in the example above is included in the
jar of htrace-core which is bundled with Hadoop.)
+```
+ $ cp htrace-htraced/target/htrace-htraced-3.2.0-incubating.jar $HADOOP_HOME/share/hadoop/common/lib/
+```
+
### Dynamic update of tracing configuration
You can use `hadoop trace` command to see and update the tracing configuration of each servers.
@@ -149,6 +153,7 @@ which start tracing span before invoking HDFS shell command.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.tracing.SpanReceiverHost;
import org.apache.hadoop.util.ToolRunner;
import org.apache.htrace.Sampler;
@@ -157,19 +162,16 @@ which start tracing span before invoking HDFS shell command.
public class TracingFsShell {
public static void main(String argv[]) throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = new HdfsConfiguration();
FsShell shell = new FsShell();
conf.setQuietMode(false);
shell.setConf(conf);
SpanReceiverHost.get(conf, DFSConfigKeys.DFS_SERVER_HTRACE_PREFIX);
int res = 0;
- TraceScope ts = null;
- try {
- ts = Trace.startSpan("FsShell", Sampler.ALWAYS);
+ try (TraceScope ts = Trace.startSpan("FsShell", Sampler.ALWAYS)) {
res = ToolRunner.run(shell, argv);
} finally {
shell.close();
- if (ts != null) ts.close();
}
System.exit(res);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 60e5577..fc1cd26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -37,7 +37,6 @@ import java.net.Socket;
import java.net.SocketAddress;
import java.net.URI;
import java.net.UnknownHostException;
-import java.nio.charset.Charset;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Collections;
@@ -3223,35 +3222,26 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
return saslClient;
}
- private static final byte[] PATH = "path".getBytes(Charset.forName("UTF-8"));
-
TraceScope getPathTraceScope(String description, String path) {
TraceScope scope = Trace.startSpan(description, traceSampler);
Span span = scope.getSpan();
if (span != null) {
if (path != null) {
- span.addKVAnnotation(PATH,
- path.getBytes(Charset.forName("UTF-8")));
+ span.addKVAnnotation("path", path);
}
}
return scope;
}
- private static final byte[] SRC = "src".getBytes(Charset.forName("UTF-8"));
-
- private static final byte[] DST = "dst".getBytes(Charset.forName("UTF-8"));
-
TraceScope getSrcDstTraceScope(String description, String src, String dst) {
TraceScope scope = Trace.startSpan(description, traceSampler);
Span span = scope.getSpan();
if (span != null) {
if (src != null) {
- span.addKVAnnotation(SRC,
- src.getBytes(Charset.forName("UTF-8")));
+ span.addKVAnnotation("src", src);
}
if (dst != null) {
- span.addKVAnnotation(DST,
- dst.getBytes(Charset.forName("UTF-8")));
+ span.addKVAnnotation("dst", dst);
}
}
return scope;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81f0adb7/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 73e29f8..94e1f5e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -820,7 +820,7 @@
<dependency>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core</artifactId>
- <version>3.1.0-incubating</version>
+ <version>3.2.0-incubating</version>
</dependency>
<dependency>
<groupId>org.jdom</groupId>
[05/50] [abbrv] hadoop git commit: HDFS-8482. Rename
BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.
Posted by zj...@apache.org.
HDFS-8482. Rename BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b88df72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b88df72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b88df72
Branch: refs/heads/YARN-2928
Commit: 8b88df723b75e8a81ed7db7599baeb4098059f46
Parents: 00c4b9d
Author: Andrew Wang <wa...@apache.org>
Authored: Wed May 27 15:37:01 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 15:46:59 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/blockmanagement/BlockCollection.java | 8 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 399 +++++++++++++++++++
.../blockmanagement/BlockInfoContiguous.java | 399 -------------------
.../BlockInfoContiguousUnderConstruction.java | 6 +-
.../server/blockmanagement/BlockManager.java | 124 +++---
.../hdfs/server/blockmanagement/BlocksMap.java | 32 +-
.../CacheReplicationMonitor.java | 14 +-
.../blockmanagement/DatanodeDescriptor.java | 16 +-
.../blockmanagement/DatanodeStorageInfo.java | 25 +-
.../blockmanagement/DecommissionManager.java | 22 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 14 +-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 8 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 14 +-
.../hdfs/server/namenode/FSImageFormat.java | 14 +-
.../server/namenode/FSImageFormatPBINode.java | 11 +-
.../server/namenode/FSImageSerialization.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 42 +-
.../namenode/FileUnderConstructionFeature.java | 6 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 74 ++--
.../hdfs/server/namenode/LeaseManager.java | 6 +-
.../hdfs/server/namenode/NamenodeFsck.java | 4 +-
.../snapshot/FSImageFormatPBSnapshot.java | 8 +-
.../hdfs/server/namenode/snapshot/FileDiff.java | 10 +-
.../server/namenode/snapshot/FileDiffList.java | 20 +-
.../snapshot/FileWithSnapshotFeature.java | 4 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +-
.../apache/hadoop/hdfs/TestDecommission.java | 4 +-
.../server/blockmanagement/TestBlockInfo.java | 20 +-
.../blockmanagement/TestBlockManager.java | 36 +-
.../blockmanagement/TestDatanodeDescriptor.java | 4 +-
.../blockmanagement/TestPendingReplication.java | 4 +-
.../blockmanagement/TestReplicationPolicy.java | 9 +-
.../hdfs/server/namenode/CreateEditsLog.java | 8 +-
.../hdfs/server/namenode/TestAddBlock.java | 12 +-
.../namenode/TestBlockUnderConstruction.java | 6 +-
.../TestCommitBlockSynchronization.java | 8 +-
.../hdfs/server/namenode/TestEditLog.java | 4 +-
.../hdfs/server/namenode/TestFSImage.java | 4 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 4 +-
.../server/namenode/TestGetBlockLocations.java | 4 +-
.../hdfs/server/namenode/TestINodeFile.java | 4 +-
.../namenode/TestTruncateQuotaUpdate.java | 16 +-
.../namenode/snapshot/SnapshotTestHelper.java | 4 +-
.../snapshot/TestFileWithSnapshotFeature.java | 6 +-
.../snapshot/TestSnapshotBlocksMap.java | 24 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 16 +-
48 files changed, 745 insertions(+), 750 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ce97552..948a516 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -580,6 +580,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8377. Support HTTP/2 in datanode. (Duo Zhang via wheat9)
+ HDFS-8482. Rename BlockInfoContiguous to BlockInfo. (Zhe Zhang via wang)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
index c0a959c..02a1d05 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
@@ -31,7 +31,7 @@ public interface BlockCollection {
/**
* Get the last block of the collection.
*/
- public BlockInfoContiguous getLastBlock();
+ public BlockInfo getLastBlock();
/**
* Get content summary.
@@ -46,7 +46,7 @@ public interface BlockCollection {
/**
* Get the blocks.
*/
- public BlockInfoContiguous[] getBlocks();
+ public BlockInfo[] getBlocks();
/**
* Get preferred block size for the collection
@@ -73,13 +73,13 @@ public interface BlockCollection {
/**
* Set the block at the given index.
*/
- public void setBlock(int index, BlockInfoContiguous blk);
+ public void setBlock(int index, BlockInfo blk);
/**
* Convert the last block of the collection to an under-construction block
* and set the locations.
*/
- public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfoContiguous lastBlock,
+ public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfo lastBlock,
DatanodeStorageInfo[] targets) throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
new file mode 100644
index 0000000..0370c57
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -0,0 +1,399 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.LinkedList;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.util.LightWeightGSet;
+
+/**
+ * BlockInfo class maintains for a given block
+ * the {@link BlockCollection} it is part of and datanodes where the replicas of
+ * the block are stored.
+ */
+@InterfaceAudience.Private
+public class BlockInfo extends Block
+ implements LightWeightGSet.LinkedElement {
+ public static final BlockInfo[] EMPTY_ARRAY = {};
+
+ private BlockCollection bc;
+
+ /** For implementing {@link LightWeightGSet.LinkedElement} interface */
+ private LightWeightGSet.LinkedElement nextLinkedElement;
+
+ /**
+ * This array contains triplets of references. For each i-th storage, the
+ * block belongs to triplets[3*i] is the reference to the
+ * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
+ * references to the previous and the next blocks, respectively, in the list
+ * of blocks belonging to this storage.
+ *
+ * Using previous and next in Object triplets is done instead of a
+ * {@link LinkedList} list to efficiently use memory. With LinkedList the cost
+ * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
+ * bytes using the triplets.
+ */
+ private Object[] triplets;
+
+ /**
+ * Construct an entry for blocksmap
+ * @param replication the block's replication factor
+ */
+ public BlockInfo(short replication) {
+ this.triplets = new Object[3*replication];
+ this.bc = null;
+ }
+
+ public BlockInfo(Block blk, short replication) {
+ super(blk);
+ this.triplets = new Object[3*replication];
+ this.bc = null;
+ }
+
+ /**
+ * Copy construction.
+ * This is used to convert BlockInfoUnderConstruction
+ * @param from BlockInfo to copy from.
+ */
+ protected BlockInfo(BlockInfo from) {
+ super(from);
+ this.triplets = new Object[from.triplets.length];
+ this.bc = from.bc;
+ }
+
+ public BlockCollection getBlockCollection() {
+ return bc;
+ }
+
+ public void setBlockCollection(BlockCollection bc) {
+ this.bc = bc;
+ }
+
+ public boolean isDeleted() {
+ return (bc == null);
+ }
+
+ public DatanodeDescriptor getDatanode(int index) {
+ DatanodeStorageInfo storage = getStorageInfo(index);
+ return storage == null ? null : storage.getDatanodeDescriptor();
+ }
+
+ DatanodeStorageInfo getStorageInfo(int index) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
+ return (DatanodeStorageInfo)triplets[index*3];
+ }
+
+ private BlockInfo getPrevious(int index) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+1];
+ assert info == null ||
+ info.getClass().getName().startsWith(BlockInfo.class.getName()) :
+ "BlockInfo is expected at " + index*3;
+ return info;
+ }
+
+ BlockInfo getNext(int index) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+2];
+ assert info == null || info.getClass().getName().startsWith(
+ BlockInfo.class.getName()) :
+ "BlockInfo is expected at " + index*3;
+ return info;
+ }
+
+ private void setStorageInfo(int index, DatanodeStorageInfo storage) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
+ triplets[index*3] = storage;
+ }
+
+ /**
+ * Return the previous block on the block list for the datanode at
+ * position index. Set the previous block on the list to "to".
+ *
+ * @param index - the datanode index
+ * @param to - block to be set to previous on the list of blocks
+ * @return current previous block on the list of blocks
+ */
+ private BlockInfo setPrevious(int index, BlockInfo to) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+1];
+ triplets[index*3+1] = to;
+ return info;
+ }
+
+ /**
+ * Return the next block on the block list for the datanode at
+ * position index. Set the next block on the list to "to".
+ *
+ * @param index - the datanode index
+ * @param to - block to be set to next on the list of blocks
+ * * @return current next block on the list of blocks
+ */
+ private BlockInfo setNext(int index, BlockInfo to) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+2];
+ triplets[index*3+2] = to;
+ return info;
+ }
+
+ public int getCapacity() {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+ return triplets.length / 3;
+ }
+
+ /**
+ * Ensure that there is enough space to include num more triplets.
+ * @return first free triplet index.
+ */
+ private int ensureCapacity(int num) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ int last = numNodes();
+ if(triplets.length >= (last+num)*3)
+ return last;
+ /* Not enough space left. Create a new array. Should normally
+ * happen only when replication is manually increased by the user. */
+ Object[] old = triplets;
+ triplets = new Object[(last+num)*3];
+ System.arraycopy(old, 0, triplets, 0, last*3);
+ return last;
+ }
+
+ /**
+ * Count the number of data-nodes the block belongs to.
+ */
+ public int numNodes() {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+ for(int idx = getCapacity()-1; idx >= 0; idx--) {
+ if(getDatanode(idx) != null)
+ return idx+1;
+ }
+ return 0;
+ }
+
+ /**
+ * Add a {@link DatanodeStorageInfo} location for a block
+ */
+ boolean addStorage(DatanodeStorageInfo storage) {
+ // find the last null node
+ int lastNode = ensureCapacity(1);
+ setStorageInfo(lastNode, storage);
+ setNext(lastNode, null);
+ setPrevious(lastNode, null);
+ return true;
+ }
+
+ /**
+ * Remove {@link DatanodeStorageInfo} location for a block
+ */
+ boolean removeStorage(DatanodeStorageInfo storage) {
+ int dnIndex = findStorageInfo(storage);
+ if(dnIndex < 0) // the node is not found
+ return false;
+ assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+ "Block is still in the list and must be removed first.";
+ // find the last not null node
+ int lastNode = numNodes()-1;
+ // replace current node triplet by the lastNode one
+ setStorageInfo(dnIndex, getStorageInfo(lastNode));
+ setNext(dnIndex, getNext(lastNode));
+ setPrevious(dnIndex, getPrevious(lastNode));
+ // set the last triplet to null
+ setStorageInfo(lastNode, null);
+ setNext(lastNode, null);
+ setPrevious(lastNode, null);
+ return true;
+ }
+
+ /**
+ * Find specified DatanodeStorageInfo.
+ * @return DatanodeStorageInfo or null if not found.
+ */
+ DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
+ int len = getCapacity();
+ for(int idx = 0; idx < len; idx++) {
+ DatanodeStorageInfo cur = getStorageInfo(idx);
+ if(cur == null)
+ break;
+ if(cur.getDatanodeDescriptor() == dn)
+ return cur;
+ }
+ return null;
+ }
+
+ /**
+ * Find specified DatanodeStorageInfo.
+ * @return index or -1 if not found.
+ */
+ int findStorageInfo(DatanodeStorageInfo storageInfo) {
+ int len = getCapacity();
+ for(int idx = 0; idx < len; idx++) {
+ DatanodeStorageInfo cur = getStorageInfo(idx);
+ if (cur == storageInfo) {
+ return idx;
+ }
+ if (cur == null) {
+ break;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Insert this block into the head of the list of blocks
+ * related to the specified DatanodeStorageInfo.
+ * If the head is null then form a new list.
+ * @return current block as the new head of the list.
+ */
+ BlockInfo listInsert(BlockInfo head,
+ DatanodeStorageInfo storage) {
+ int dnIndex = this.findStorageInfo(storage);
+ assert dnIndex >= 0 : "Data node is not found: current";
+ assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+ "Block is already in the list and cannot be inserted.";
+ this.setPrevious(dnIndex, null);
+ this.setNext(dnIndex, head);
+ if(head != null)
+ head.setPrevious(head.findStorageInfo(storage), this);
+ return this;
+ }
+
+ /**
+ * Remove this block from the list of blocks
+ * related to the specified DatanodeStorageInfo.
+ * If this block is the head of the list then return the next block as
+ * the new head.
+ * @return the new head of the list or null if the list becomes
+ * empy after deletion.
+ */
+ BlockInfo listRemove(BlockInfo head,
+ DatanodeStorageInfo storage) {
+ if(head == null)
+ return null;
+ int dnIndex = this.findStorageInfo(storage);
+ if(dnIndex < 0) // this block is not on the data-node list
+ return head;
+
+ BlockInfo next = this.getNext(dnIndex);
+ BlockInfo prev = this.getPrevious(dnIndex);
+ this.setNext(dnIndex, null);
+ this.setPrevious(dnIndex, null);
+ if(prev != null)
+ prev.setNext(prev.findStorageInfo(storage), next);
+ if(next != null)
+ next.setPrevious(next.findStorageInfo(storage), prev);
+ if(this == head) // removing the head
+ head = next;
+ return head;
+ }
+
+ /**
+ * Remove this block from the list of blocks related to the specified
+ * DatanodeDescriptor. Insert it into the head of the list of blocks.
+ *
+ * @return the new head of the list.
+ */
+ public BlockInfo moveBlockToHead(BlockInfo head,
+ DatanodeStorageInfo storage, int curIndex, int headIndex) {
+ if (head == this) {
+ return this;
+ }
+ BlockInfo next = this.setNext(curIndex, head);
+ BlockInfo prev = this.setPrevious(curIndex, null);
+
+ head.setPrevious(headIndex, this);
+ prev.setNext(prev.findStorageInfo(storage), next);
+ if (next != null) {
+ next.setPrevious(next.findStorageInfo(storage), prev);
+ }
+ return this;
+ }
+
+ /**
+ * BlockInfo represents a block that is not being constructed.
+ * In order to start modifying the block, the BlockInfo should be converted
+ * to {@link BlockInfoContiguousUnderConstruction}.
+ * @return {@link BlockUCState#COMPLETE}
+ */
+ public BlockUCState getBlockUCState() {
+ return BlockUCState.COMPLETE;
+ }
+
+ /**
+ * Is this block complete?
+ *
+ * @return true if the state of the block is {@link BlockUCState#COMPLETE}
+ */
+ public boolean isComplete() {
+ return getBlockUCState().equals(BlockUCState.COMPLETE);
+ }
+
+ /**
+ * Convert a complete block to an under construction block.
+ * @return BlockInfoUnderConstruction - an under construction block.
+ */
+ public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
+ BlockUCState s, DatanodeStorageInfo[] targets) {
+ if(isComplete()) {
+ BlockInfoContiguousUnderConstruction ucBlock =
+ new BlockInfoContiguousUnderConstruction(this,
+ getBlockCollection().getPreferredBlockReplication(), s, targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ return ucBlock;
+ }
+ // the block is already under construction
+ BlockInfoContiguousUnderConstruction ucBlock =
+ (BlockInfoContiguousUnderConstruction)this;
+ ucBlock.setBlockUCState(s);
+ ucBlock.setExpectedLocations(targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ return ucBlock;
+ }
+
+ @Override
+ public int hashCode() {
+ // Super implementation is sufficient
+ return super.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // Sufficient to rely on super's implementation
+ return (this == obj) || super.equals(obj);
+ }
+
+ @Override
+ public LightWeightGSet.LinkedElement getNext() {
+ return nextLinkedElement;
+ }
+
+ @Override
+ public void setNext(LightWeightGSet.LinkedElement next) {
+ this.nextLinkedElement = next;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
deleted file mode 100644
index 769046b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ /dev/null
@@ -1,399 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.blockmanagement;
-
-import java.util.LinkedList;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import org.apache.hadoop.util.LightWeightGSet;
-
-/**
- * BlockInfo class maintains for a given block
- * the {@link BlockCollection} it is part of and datanodes where the replicas of
- * the block are stored.
- */
-@InterfaceAudience.Private
-public class BlockInfoContiguous extends Block
- implements LightWeightGSet.LinkedElement {
- public static final BlockInfoContiguous[] EMPTY_ARRAY = {};
-
- private BlockCollection bc;
-
- /** For implementing {@link LightWeightGSet.LinkedElement} interface */
- private LightWeightGSet.LinkedElement nextLinkedElement;
-
- /**
- * This array contains triplets of references. For each i-th storage, the
- * block belongs to triplets[3*i] is the reference to the
- * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
- * references to the previous and the next blocks, respectively, in the list
- * of blocks belonging to this storage.
- *
- * Using previous and next in Object triplets is done instead of a
- * {@link LinkedList} list to efficiently use memory. With LinkedList the cost
- * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
- * bytes using the triplets.
- */
- private Object[] triplets;
-
- /**
- * Construct an entry for blocksmap
- * @param replication the block's replication factor
- */
- public BlockInfoContiguous(short replication) {
- this.triplets = new Object[3*replication];
- this.bc = null;
- }
-
- public BlockInfoContiguous(Block blk, short replication) {
- super(blk);
- this.triplets = new Object[3*replication];
- this.bc = null;
- }
-
- /**
- * Copy construction.
- * This is used to convert BlockInfoUnderConstruction
- * @param from BlockInfo to copy from.
- */
- protected BlockInfoContiguous(BlockInfoContiguous from) {
- super(from);
- this.triplets = new Object[from.triplets.length];
- this.bc = from.bc;
- }
-
- public BlockCollection getBlockCollection() {
- return bc;
- }
-
- public void setBlockCollection(BlockCollection bc) {
- this.bc = bc;
- }
-
- public boolean isDeleted() {
- return (bc == null);
- }
-
- public DatanodeDescriptor getDatanode(int index) {
- DatanodeStorageInfo storage = getStorageInfo(index);
- return storage == null ? null : storage.getDatanodeDescriptor();
- }
-
- DatanodeStorageInfo getStorageInfo(int index) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
- return (DatanodeStorageInfo)triplets[index*3];
- }
-
- private BlockInfoContiguous getPrevious(int index) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
- BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
- assert info == null ||
- info.getClass().getName().startsWith(BlockInfoContiguous.class.getName()) :
- "BlockInfo is expected at " + index*3;
- return info;
- }
-
- BlockInfoContiguous getNext(int index) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
- BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
- assert info == null || info.getClass().getName().startsWith(
- BlockInfoContiguous.class.getName()) :
- "BlockInfo is expected at " + index*3;
- return info;
- }
-
- private void setStorageInfo(int index, DatanodeStorageInfo storage) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
- triplets[index*3] = storage;
- }
-
- /**
- * Return the previous block on the block list for the datanode at
- * position index. Set the previous block on the list to "to".
- *
- * @param index - the datanode index
- * @param to - block to be set to previous on the list of blocks
- * @return current previous block on the list of blocks
- */
- private BlockInfoContiguous setPrevious(int index, BlockInfoContiguous to) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
- BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
- triplets[index*3+1] = to;
- return info;
- }
-
- /**
- * Return the next block on the block list for the datanode at
- * position index. Set the next block on the list to "to".
- *
- * @param index - the datanode index
- * @param to - block to be set to next on the list of blocks
- * * @return current next block on the list of blocks
- */
- private BlockInfoContiguous setNext(int index, BlockInfoContiguous to) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
- BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
- triplets[index*3+2] = to;
- return info;
- }
-
- public int getCapacity() {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert triplets.length % 3 == 0 : "Malformed BlockInfo";
- return triplets.length / 3;
- }
-
- /**
- * Ensure that there is enough space to include num more triplets.
- * @return first free triplet index.
- */
- private int ensureCapacity(int num) {
- assert this.triplets != null : "BlockInfo is not initialized";
- int last = numNodes();
- if(triplets.length >= (last+num)*3)
- return last;
- /* Not enough space left. Create a new array. Should normally
- * happen only when replication is manually increased by the user. */
- Object[] old = triplets;
- triplets = new Object[(last+num)*3];
- System.arraycopy(old, 0, triplets, 0, last*3);
- return last;
- }
-
- /**
- * Count the number of data-nodes the block belongs to.
- */
- public int numNodes() {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert triplets.length % 3 == 0 : "Malformed BlockInfo";
- for(int idx = getCapacity()-1; idx >= 0; idx--) {
- if(getDatanode(idx) != null)
- return idx+1;
- }
- return 0;
- }
-
- /**
- * Add a {@link DatanodeStorageInfo} location for a block
- */
- boolean addStorage(DatanodeStorageInfo storage) {
- // find the last null node
- int lastNode = ensureCapacity(1);
- setStorageInfo(lastNode, storage);
- setNext(lastNode, null);
- setPrevious(lastNode, null);
- return true;
- }
-
- /**
- * Remove {@link DatanodeStorageInfo} location for a block
- */
- boolean removeStorage(DatanodeStorageInfo storage) {
- int dnIndex = findStorageInfo(storage);
- if(dnIndex < 0) // the node is not found
- return false;
- assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
- "Block is still in the list and must be removed first.";
- // find the last not null node
- int lastNode = numNodes()-1;
- // replace current node triplet by the lastNode one
- setStorageInfo(dnIndex, getStorageInfo(lastNode));
- setNext(dnIndex, getNext(lastNode));
- setPrevious(dnIndex, getPrevious(lastNode));
- // set the last triplet to null
- setStorageInfo(lastNode, null);
- setNext(lastNode, null);
- setPrevious(lastNode, null);
- return true;
- }
-
- /**
- * Find specified DatanodeStorageInfo.
- * @return DatanodeStorageInfo or null if not found.
- */
- DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
- int len = getCapacity();
- for(int idx = 0; idx < len; idx++) {
- DatanodeStorageInfo cur = getStorageInfo(idx);
- if(cur == null)
- break;
- if(cur.getDatanodeDescriptor() == dn)
- return cur;
- }
- return null;
- }
-
- /**
- * Find specified DatanodeStorageInfo.
- * @return index or -1 if not found.
- */
- int findStorageInfo(DatanodeStorageInfo storageInfo) {
- int len = getCapacity();
- for(int idx = 0; idx < len; idx++) {
- DatanodeStorageInfo cur = getStorageInfo(idx);
- if (cur == storageInfo) {
- return idx;
- }
- if (cur == null) {
- break;
- }
- }
- return -1;
- }
-
- /**
- * Insert this block into the head of the list of blocks
- * related to the specified DatanodeStorageInfo.
- * If the head is null then form a new list.
- * @return current block as the new head of the list.
- */
- BlockInfoContiguous listInsert(BlockInfoContiguous head,
- DatanodeStorageInfo storage) {
- int dnIndex = this.findStorageInfo(storage);
- assert dnIndex >= 0 : "Data node is not found: current";
- assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
- "Block is already in the list and cannot be inserted.";
- this.setPrevious(dnIndex, null);
- this.setNext(dnIndex, head);
- if(head != null)
- head.setPrevious(head.findStorageInfo(storage), this);
- return this;
- }
-
- /**
- * Remove this block from the list of blocks
- * related to the specified DatanodeStorageInfo.
- * If this block is the head of the list then return the next block as
- * the new head.
- * @return the new head of the list or null if the list becomes
- * empy after deletion.
- */
- BlockInfoContiguous listRemove(BlockInfoContiguous head,
- DatanodeStorageInfo storage) {
- if(head == null)
- return null;
- int dnIndex = this.findStorageInfo(storage);
- if(dnIndex < 0) // this block is not on the data-node list
- return head;
-
- BlockInfoContiguous next = this.getNext(dnIndex);
- BlockInfoContiguous prev = this.getPrevious(dnIndex);
- this.setNext(dnIndex, null);
- this.setPrevious(dnIndex, null);
- if(prev != null)
- prev.setNext(prev.findStorageInfo(storage), next);
- if(next != null)
- next.setPrevious(next.findStorageInfo(storage), prev);
- if(this == head) // removing the head
- head = next;
- return head;
- }
-
- /**
- * Remove this block from the list of blocks related to the specified
- * DatanodeDescriptor. Insert it into the head of the list of blocks.
- *
- * @return the new head of the list.
- */
- public BlockInfoContiguous moveBlockToHead(BlockInfoContiguous head,
- DatanodeStorageInfo storage, int curIndex, int headIndex) {
- if (head == this) {
- return this;
- }
- BlockInfoContiguous next = this.setNext(curIndex, head);
- BlockInfoContiguous prev = this.setPrevious(curIndex, null);
-
- head.setPrevious(headIndex, this);
- prev.setNext(prev.findStorageInfo(storage), next);
- if (next != null) {
- next.setPrevious(next.findStorageInfo(storage), prev);
- }
- return this;
- }
-
- /**
- * BlockInfo represents a block that is not being constructed.
- * In order to start modifying the block, the BlockInfo should be converted
- * to {@link BlockInfoContiguousUnderConstruction}.
- * @return {@link BlockUCState#COMPLETE}
- */
- public BlockUCState getBlockUCState() {
- return BlockUCState.COMPLETE;
- }
-
- /**
- * Is this block complete?
- *
- * @return true if the state of the block is {@link BlockUCState#COMPLETE}
- */
- public boolean isComplete() {
- return getBlockUCState().equals(BlockUCState.COMPLETE);
- }
-
- /**
- * Convert a complete block to an under construction block.
- * @return BlockInfoUnderConstruction - an under construction block.
- */
- public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
- BlockUCState s, DatanodeStorageInfo[] targets) {
- if(isComplete()) {
- BlockInfoContiguousUnderConstruction ucBlock =
- new BlockInfoContiguousUnderConstruction(this,
- getBlockCollection().getPreferredBlockReplication(), s, targets);
- ucBlock.setBlockCollection(getBlockCollection());
- return ucBlock;
- }
- // the block is already under construction
- BlockInfoContiguousUnderConstruction ucBlock =
- (BlockInfoContiguousUnderConstruction)this;
- ucBlock.setBlockUCState(s);
- ucBlock.setExpectedLocations(targets);
- ucBlock.setBlockCollection(getBlockCollection());
- return ucBlock;
- }
-
- @Override
- public int hashCode() {
- // Super implementation is sufficient
- return super.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- // Sufficient to rely on super's implementation
- return (this == obj) || super.equals(obj);
- }
-
- @Override
- public LightWeightGSet.LinkedElement getNext() {
- return nextLinkedElement;
- }
-
- @Override
- public void setNext(LightWeightGSet.LinkedElement next) {
- this.nextLinkedElement = next;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
index 92153ab..58020a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
* Represents a block that is currently being constructed.<br>
* This is usually the last block of a file opened for write or append.
*/
-public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
+public class BlockInfoContiguousUnderConstruction extends BlockInfo {
/** Block state. See {@link BlockUCState} */
private BlockUCState blockUCState;
@@ -182,10 +182,10 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
* the client or it does not have at least a minimal number of replicas
* reported from data-nodes.
*/
- BlockInfoContiguous convertToCompleteBlock() throws IOException {
+ BlockInfo convertToCompleteBlock() throws IOException {
assert getBlockUCState() != BlockUCState.COMPLETE :
"Trying to convert a COMPLETE block";
- return new BlockInfoContiguous(this);
+ return new BlockInfo(this);
}
/** Set expected locations */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 54981fb..45a485b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -543,8 +543,8 @@ public class BlockManager {
int usableReplicas = numReplicas.liveReplicas() +
numReplicas.decommissionedAndDecommissioning();
- if (block instanceof BlockInfoContiguous) {
- BlockCollection bc = ((BlockInfoContiguous) block).getBlockCollection();
+ if (block instanceof BlockInfo) {
+ BlockCollection bc = ((BlockInfo) block).getBlockCollection();
String fileName = (bc == null) ? "[orphaned]" : bc.getName();
out.print(fileName + ": ");
}
@@ -624,7 +624,7 @@ public class BlockManager {
Block commitBlock) throws IOException {
if(commitBlock == null)
return false; // not committing, this is a block allocation retry
- BlockInfoContiguous lastBlock = bc.getLastBlock();
+ BlockInfo lastBlock = bc.getLastBlock();
if(lastBlock == null)
return false; // no blocks in file yet
if(lastBlock.isComplete())
@@ -644,11 +644,11 @@ public class BlockManager {
* @throws IOException if the block does not have at least a minimal number
* of replicas reported from data-nodes.
*/
- private BlockInfoContiguous completeBlock(final BlockCollection bc,
+ private BlockInfo completeBlock(final BlockCollection bc,
final int blkIndex, boolean force) throws IOException {
if(blkIndex < 0)
return null;
- BlockInfoContiguous curBlock = bc.getBlocks()[blkIndex];
+ BlockInfo curBlock = bc.getBlocks()[blkIndex];
if(curBlock.isComplete())
return curBlock;
BlockInfoContiguousUnderConstruction ucBlock =
@@ -660,7 +660,7 @@ public class BlockManager {
if(!force && ucBlock.getBlockUCState() != BlockUCState.COMMITTED)
throw new IOException(
"Cannot complete block: block has not been COMMITTED by the client");
- BlockInfoContiguous completeBlock = ucBlock.convertToCompleteBlock();
+ BlockInfo completeBlock = ucBlock.convertToCompleteBlock();
// replace penultimate block in file
bc.setBlock(blkIndex, completeBlock);
@@ -678,9 +678,9 @@ public class BlockManager {
return blocksMap.replaceBlock(completeBlock);
}
- private BlockInfoContiguous completeBlock(final BlockCollection bc,
- final BlockInfoContiguous block, boolean force) throws IOException {
- BlockInfoContiguous[] fileBlocks = bc.getBlocks();
+ private BlockInfo completeBlock(final BlockCollection bc,
+ final BlockInfo block, boolean force) throws IOException {
+ BlockInfo[] fileBlocks = bc.getBlocks();
for(int idx = 0; idx < fileBlocks.length; idx++)
if(fileBlocks[idx] == block) {
return completeBlock(bc, idx, force);
@@ -693,7 +693,7 @@ public class BlockManager {
* regardless of whether enough replicas are present. This is necessary
* when tailing edit logs as a Standby.
*/
- public BlockInfoContiguous forceCompleteBlock(final BlockCollection bc,
+ public BlockInfo forceCompleteBlock(final BlockCollection bc,
final BlockInfoContiguousUnderConstruction block) throws IOException {
block.commitBlock(block);
return completeBlock(bc, block, true);
@@ -716,7 +716,7 @@ public class BlockManager {
*/
public LocatedBlock convertLastBlockToUnderConstruction(
BlockCollection bc, long bytesToRemove) throws IOException {
- BlockInfoContiguous oldBlock = bc.getLastBlock();
+ BlockInfo oldBlock = bc.getLastBlock();
if(oldBlock == null ||
bc.getPreferredBlockSize() == oldBlock.getNumBytes() - bytesToRemove)
return null;
@@ -769,7 +769,7 @@ public class BlockManager {
}
private List<LocatedBlock> createLocatedBlockList(
- final BlockInfoContiguous[] blocks,
+ final BlockInfo[] blocks,
final long offset, final long length, final int nrBlocksToReturn,
final AccessMode mode) throws IOException {
int curBlk = 0;
@@ -799,7 +799,7 @@ public class BlockManager {
return results;
}
- private LocatedBlock createLocatedBlock(final BlockInfoContiguous[] blocks,
+ private LocatedBlock createLocatedBlock(final BlockInfo[] blocks,
final long endPos, final AccessMode mode) throws IOException {
int curBlk = 0;
long curPos = 0;
@@ -815,7 +815,7 @@ public class BlockManager {
return createLocatedBlock(blocks[curBlk], curPos, mode);
}
- private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos,
+ private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos,
final AccessMode mode) throws IOException {
final LocatedBlock lb = createLocatedBlock(blk, pos);
if (mode != null) {
@@ -825,7 +825,7 @@ public class BlockManager {
}
/** @return a LocatedBlock for the given block */
- private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos
+ private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos
) throws IOException {
if (blk instanceof BlockInfoContiguousUnderConstruction) {
if (blk.isComplete()) {
@@ -874,7 +874,7 @@ public class BlockManager {
}
/** Create a LocatedBlocks. */
- public LocatedBlocks createLocatedBlocks(final BlockInfoContiguous[] blocks,
+ public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks,
final long fileSizeExcludeBlocksUnderConstruction,
final boolean isFileUnderConstruction, final long offset,
final long length, final boolean needBlockToken,
@@ -897,7 +897,7 @@ public class BlockManager {
final LocatedBlock lastlb;
final boolean isComplete;
if (!inSnapshot) {
- final BlockInfoContiguous last = blocks[blocks.length - 1];
+ final BlockInfo last = blocks[blocks.length - 1];
final long lastPos = last.isComplete()?
fileSizeExcludeBlocksUnderConstruction - last.getNumBytes()
: fileSizeExcludeBlocksUnderConstruction;
@@ -986,7 +986,7 @@ public class BlockManager {
/**
* Check if a block is replicated to at least the minimum replication.
*/
- public boolean isSufficientlyReplicated(BlockInfoContiguous b) {
+ public boolean isSufficientlyReplicated(BlockInfo b) {
// Compare against the lesser of the minReplication and number of live DNs.
final int replication =
Math.min(minReplication, getDatanodeManager().getNumLiveDataNodes());
@@ -1027,7 +1027,7 @@ public class BlockManager {
if(numBlocks == 0) {
return new BlocksWithLocations(new BlockWithLocations[0]);
}
- Iterator<BlockInfoContiguous> iter = node.getBlockIterator();
+ Iterator<BlockInfo> iter = node.getBlockIterator();
// starting from a random block
int startBlock = ThreadLocalRandom.current().nextInt(numBlocks);
// skip blocks
@@ -1036,7 +1036,7 @@ public class BlockManager {
}
List<BlockWithLocations> results = new ArrayList<BlockWithLocations>();
long totalSize = 0;
- BlockInfoContiguous curBlock;
+ BlockInfo curBlock;
while(totalSize<size && iter.hasNext()) {
curBlock = iter.next();
if(!curBlock.isComplete()) continue;
@@ -1135,7 +1135,7 @@ public class BlockManager {
public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk,
final DatanodeInfo dn, String storageID, String reason) throws IOException {
assert namesystem.hasWriteLock();
- final BlockInfoContiguous storedBlock = getStoredBlock(blk.getLocalBlock());
+ final BlockInfo storedBlock = getStoredBlock(blk.getLocalBlock());
if (storedBlock == null) {
// Check if the replica is in the blockMap, if not
// ignore the request for now. This could happen when BlockScanner
@@ -1694,7 +1694,7 @@ public class BlockManager {
* Use the blockinfo from the blocksmap to be certain we're working
* with the most up-to-date block information (e.g. genstamp).
*/
- BlockInfoContiguous bi = blocksMap.getStoredBlock(timedOutItems[i]);
+ BlockInfo bi = blocksMap.getStoredBlock(timedOutItems[i]);
if (bi == null) {
continue;
}
@@ -1738,16 +1738,16 @@ public class BlockManager {
*/
private static class BlockToMarkCorrupt {
/** The corrupted block in a datanode. */
- final BlockInfoContiguous corrupted;
+ final BlockInfo corrupted;
/** The corresponding block stored in the BlockManager. */
- final BlockInfoContiguous stored;
+ final BlockInfo stored;
/** The reason to mark corrupt. */
final String reason;
/** The reason code to be stored */
final Reason reasonCode;
- BlockToMarkCorrupt(BlockInfoContiguous corrupted,
- BlockInfoContiguous stored, String reason,
+ BlockToMarkCorrupt(BlockInfo corrupted,
+ BlockInfo stored, String reason,
Reason reasonCode) {
Preconditions.checkNotNull(corrupted, "corrupted is null");
Preconditions.checkNotNull(stored, "stored is null");
@@ -1758,14 +1758,14 @@ public class BlockManager {
this.reasonCode = reasonCode;
}
- BlockToMarkCorrupt(BlockInfoContiguous stored, String reason,
+ BlockToMarkCorrupt(BlockInfo stored, String reason,
Reason reasonCode) {
this(stored, stored, reason, reasonCode);
}
- BlockToMarkCorrupt(BlockInfoContiguous stored, long gs, String reason,
+ BlockToMarkCorrupt(BlockInfo stored, long gs, String reason,
Reason reasonCode) {
- this(new BlockInfoContiguous(stored), stored, reason, reasonCode);
+ this(new BlockInfo(stored), stored, reason, reasonCode);
//the corrupted block in datanode has a different generation stamp
corrupted.setGenerationStamp(gs);
}
@@ -1882,10 +1882,10 @@ public class BlockManager {
"longer exists on the DataNode.",
Long.toHexString(context.getReportId()), zombie.getStorageID());
assert(namesystem.hasWriteLock());
- Iterator<BlockInfoContiguous> iter = zombie.getBlockIterator();
+ Iterator<BlockInfo> iter = zombie.getBlockIterator();
int prevBlocks = zombie.numBlocks();
while (iter.hasNext()) {
- BlockInfoContiguous block = iter.next();
+ BlockInfo block = iter.next();
// We assume that a block can be on only one storage in a DataNode.
// That's why we pass in the DatanodeDescriptor rather than the
// DatanodeStorageInfo.
@@ -1943,7 +1943,7 @@ public class BlockManager {
break;
}
- BlockInfoContiguous bi = blocksMap.getStoredBlock(b);
+ BlockInfo bi = blocksMap.getStoredBlock(b);
if (bi == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
@@ -1983,7 +1983,7 @@ public class BlockManager {
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
//
- Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>();
+ Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>();
Collection<Block> toRemove = new TreeSet<Block>();
Collection<Block> toInvalidate = new LinkedList<Block>();
Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>();
@@ -2000,7 +2000,7 @@ public class BlockManager {
removeStoredBlock(b, node);
}
int numBlocksLogged = 0;
- for (BlockInfoContiguous b : toAdd) {
+ for (BlockInfo b : toAdd) {
addStoredBlock(b, storageInfo, null, numBlocksLogged < maxNumBlocksToLog);
numBlocksLogged++;
}
@@ -2022,7 +2022,7 @@ public class BlockManager {
* Mark block replicas as corrupt except those on the storages in
* newStorages list.
*/
- public void markBlockReplicasAsCorrupt(BlockInfoContiguous block,
+ public void markBlockReplicasAsCorrupt(BlockInfo block,
long oldGenerationStamp, long oldNumBytes,
DatanodeStorageInfo[] newStorages) throws IOException {
assert namesystem.hasWriteLock();
@@ -2091,7 +2091,7 @@ public class BlockManager {
continue;
}
- BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk);
+ BlockInfo storedBlock = blocksMap.getStoredBlock(iblk);
// If block does not belong to any file, we are done.
if (storedBlock == null) continue;
@@ -2136,7 +2136,7 @@ public class BlockManager {
private void reportDiff(DatanodeStorageInfo storageInfo,
BlockListAsLongs newReport,
- Collection<BlockInfoContiguous> toAdd, // add to DatanodeDescriptor
+ Collection<BlockInfo> toAdd, // add to DatanodeDescriptor
Collection<Block> toRemove, // remove from DatanodeDescriptor
Collection<Block> toInvalidate, // should be removed from DN
Collection<BlockToMarkCorrupt> toCorrupt, // add to corrupt replicas list
@@ -2144,7 +2144,7 @@ public class BlockManager {
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
- BlockInfoContiguous delimiter = new BlockInfoContiguous(new Block(), (short) 1);
+ BlockInfo delimiter = new BlockInfo(new Block(), (short) 1);
AddBlockResult result = storageInfo.addBlock(delimiter);
assert result == AddBlockResult.ADDED
: "Delimiting block cannot be present in the node";
@@ -2157,7 +2157,7 @@ public class BlockManager {
// scan the report and process newly reported blocks
for (BlockReportReplica iblk : newReport) {
ReplicaState iState = iblk.getState();
- BlockInfoContiguous storedBlock = processReportedBlock(storageInfo,
+ BlockInfo storedBlock = processReportedBlock(storageInfo,
iblk, iState, toAdd, toInvalidate, toCorrupt, toUC);
// move block to the head of the list
@@ -2169,7 +2169,7 @@ public class BlockManager {
// collect blocks that have not been reported
// all of them are next to the delimiter
- Iterator<BlockInfoContiguous> it =
+ Iterator<BlockInfo> it =
storageInfo.new BlockIterator(delimiter.getNext(0));
while(it.hasNext())
toRemove.add(it.next());
@@ -2207,10 +2207,10 @@ public class BlockManager {
* @return the up-to-date stored block, if it should be kept.
* Otherwise, null.
*/
- private BlockInfoContiguous processReportedBlock(
+ private BlockInfo processReportedBlock(
final DatanodeStorageInfo storageInfo,
final Block block, final ReplicaState reportedState,
- final Collection<BlockInfoContiguous> toAdd,
+ final Collection<BlockInfo> toAdd,
final Collection<Block> toInvalidate,
final Collection<BlockToMarkCorrupt> toCorrupt,
final Collection<StatefulBlockInfo> toUC) {
@@ -2231,7 +2231,7 @@ public class BlockManager {
}
// find block by blockId
- BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block);
+ BlockInfo storedBlock = blocksMap.getStoredBlock(block);
if(storedBlock == null) {
// If blocksMap does not contain reported block id,
// the replica should be removed from the data-node.
@@ -2370,7 +2370,7 @@ public class BlockManager {
*/
private BlockToMarkCorrupt checkReplicaCorrupt(
Block reported, ReplicaState reportedState,
- BlockInfoContiguous storedBlock, BlockUCState ucState,
+ BlockInfo storedBlock, BlockUCState ucState,
DatanodeDescriptor dn) {
switch(reportedState) {
case FINALIZED:
@@ -2443,7 +2443,7 @@ public class BlockManager {
}
}
- private boolean isBlockUnderConstruction(BlockInfoContiguous storedBlock,
+ private boolean isBlockUnderConstruction(BlockInfo storedBlock,
BlockUCState ucState, ReplicaState reportedState) {
switch(reportedState) {
case FINALIZED:
@@ -2487,7 +2487,7 @@ public class BlockManager {
*
* @throws IOException
*/
- private void addStoredBlockImmediate(BlockInfoContiguous storedBlock,
+ private void addStoredBlockImmediate(BlockInfo storedBlock,
DatanodeStorageInfo storageInfo)
throws IOException {
assert (storedBlock != null && namesystem.hasWriteLock());
@@ -2519,13 +2519,13 @@ public class BlockManager {
* needed replications if this takes care of the problem.
* @return the block that is stored in blockMap.
*/
- private Block addStoredBlock(final BlockInfoContiguous block,
+ private Block addStoredBlock(final BlockInfo block,
DatanodeStorageInfo storageInfo,
DatanodeDescriptor delNodeHint,
boolean logEveryBlock)
throws IOException {
assert block != null && namesystem.hasWriteLock();
- BlockInfoContiguous storedBlock;
+ BlockInfo storedBlock;
DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
if (block instanceof BlockInfoContiguousUnderConstruction) {
//refresh our copy in case the block got completed in another thread
@@ -2623,7 +2623,7 @@ public class BlockManager {
return storedBlock;
}
- private void logAddStoredBlock(BlockInfoContiguous storedBlock,
+ private void logAddStoredBlock(BlockInfo storedBlock,
DatanodeDescriptor node) {
if (!blockLog.isInfoEnabled()) {
return;
@@ -2651,7 +2651,7 @@ public class BlockManager {
*
* @param blk Block whose corrupt replicas need to be invalidated
*/
- private void invalidateCorruptReplicas(BlockInfoContiguous blk) {
+ private void invalidateCorruptReplicas(BlockInfo blk) {
Collection<DatanodeDescriptor> nodes = corruptReplicas.getNodes(blk);
boolean removedFromBlocksMap = true;
if (nodes == null)
@@ -2730,7 +2730,7 @@ public class BlockManager {
long nrInvalid = 0, nrOverReplicated = 0;
long nrUnderReplicated = 0, nrPostponed = 0, nrUnderConstruction = 0;
long startTimeMisReplicatedScan = Time.monotonicNow();
- Iterator<BlockInfoContiguous> blocksItr = blocksMap.getBlocks().iterator();
+ Iterator<BlockInfo> blocksItr = blocksMap.getBlocks().iterator();
long totalBlocks = blocksMap.size();
replicationQueuesInitProgress = 0;
long totalProcessed = 0;
@@ -2742,7 +2742,7 @@ public class BlockManager {
namesystem.writeLockInterruptibly();
try {
while (processed < numBlocksPerIteration && blocksItr.hasNext()) {
- BlockInfoContiguous block = blocksItr.next();
+ BlockInfo block = blocksItr.next();
MisReplicationResult res = processMisReplicatedBlock(block);
if (LOG.isTraceEnabled()) {
LOG.trace("block " + block + ": " + res);
@@ -2817,7 +2817,7 @@ public class BlockManager {
* appropriate queues if necessary, and returns a result code indicating
* what happened with it.
*/
- private MisReplicationResult processMisReplicatedBlock(BlockInfoContiguous block) {
+ private MisReplicationResult processMisReplicatedBlock(BlockInfo block) {
if (block.isDeleted()) {
// block does not belong to any file
addToInvalidates(block);
@@ -3157,7 +3157,7 @@ public class BlockManager {
ReplicaState reportedState, DatanodeDescriptor delHintNode)
throws IOException {
// blockReceived reports a finalized block
- Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>();
+ Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>();
Collection<Block> toInvalidate = new LinkedList<Block>();
Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>();
Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>();
@@ -3174,7 +3174,7 @@ public class BlockManager {
addStoredBlockUnderConstruction(b, storageInfo);
}
long numBlocksLogged = 0;
- for (BlockInfoContiguous b : toAdd) {
+ for (BlockInfo b : toAdd) {
addStoredBlock(b, storageInfo, delHintNode, numBlocksLogged < maxNumBlocksToLog);
numBlocksLogged++;
}
@@ -3301,7 +3301,7 @@ public class BlockManager {
* @param b - the block being tested
* @return count of live nodes for this block
*/
- int countLiveNodes(BlockInfoContiguous b) {
+ int countLiveNodes(BlockInfo b) {
if (!namesystem.isInStartupSafeMode()) {
return countNodes(b).liveReplicas();
}
@@ -3380,7 +3380,7 @@ public class BlockManager {
return blocksMap.size();
}
- public DatanodeStorageInfo[] getStorages(BlockInfoContiguous block) {
+ public DatanodeStorageInfo[] getStorages(BlockInfo block) {
final DatanodeStorageInfo[] storages = new DatanodeStorageInfo[block.numNodes()];
int i = 0;
for(DatanodeStorageInfo s : blocksMap.getStorages(block)) {
@@ -3409,7 +3409,7 @@ public class BlockManager {
}
}
- public BlockInfoContiguous getStoredBlock(Block block) {
+ public BlockInfo getStoredBlock(Block block) {
return blocksMap.getStoredBlock(block);
}
@@ -3462,8 +3462,8 @@ public class BlockManager {
* replicated.
*/
public boolean checkBlocksProperlyReplicated(
- String src, BlockInfoContiguous[] blocks) {
- for (BlockInfoContiguous b: blocks) {
+ String src, BlockInfo[] blocks) {
+ for (BlockInfo b: blocks) {
if (!b.isComplete()) {
final BlockInfoContiguousUnderConstruction uc =
(BlockInfoContiguousUnderConstruction)b;
@@ -3578,7 +3578,7 @@ public class BlockManager {
return this.neededReplications.getCorruptReplOneBlockSize();
}
- public BlockInfoContiguous addBlockCollection(BlockInfoContiguous block,
+ public BlockInfo addBlockCollection(BlockInfo block,
BlockCollection bc) {
return blocksMap.addBlockCollection(block, bc);
}
@@ -3818,7 +3818,7 @@ public class BlockManager {
/**
* A simple result enum for the result of
- * {@link BlockManager#processMisReplicatedBlock(BlockInfoContiguous)}.
+ * {@link BlockManager#processMisReplicatedBlock(BlockInfo)}.
*/
enum MisReplicationResult {
/** The block should be invalidated since it belongs to a deleted file. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index 5e7d34f..d06d503 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -36,10 +36,10 @@ import com.google.common.collect.Iterables;
*/
class BlocksMap {
private static class StorageIterator implements Iterator<DatanodeStorageInfo> {
- private final BlockInfoContiguous blockInfo;
+ private final BlockInfo blockInfo;
private int nextIdx = 0;
- StorageIterator(BlockInfoContiguous blkInfo) {
+ StorageIterator(BlockInfo blkInfo) {
this.blockInfo = blkInfo;
}
@@ -63,14 +63,14 @@ class BlocksMap {
/** Constant {@link LightWeightGSet} capacity. */
private final int capacity;
- private GSet<Block, BlockInfoContiguous> blocks;
+ private GSet<Block, BlockInfo> blocks;
BlocksMap(int capacity) {
// Use 2% of total memory to size the GSet capacity
this.capacity = capacity;
- this.blocks = new LightWeightGSet<Block, BlockInfoContiguous>(capacity) {
+ this.blocks = new LightWeightGSet<Block, BlockInfo>(capacity) {
@Override
- public Iterator<BlockInfoContiguous> iterator() {
+ public Iterator<BlockInfo> iterator() {
SetIterator iterator = new SetIterator();
/*
* Not tracking any modifications to set. As this set will be used
@@ -97,15 +97,15 @@ class BlocksMap {
}
BlockCollection getBlockCollection(Block b) {
- BlockInfoContiguous info = blocks.get(b);
+ BlockInfo info = blocks.get(b);
return (info != null) ? info.getBlockCollection() : null;
}
/**
* Add block b belonging to the specified block collection to the map.
*/
- BlockInfoContiguous addBlockCollection(BlockInfoContiguous b, BlockCollection bc) {
- BlockInfoContiguous info = blocks.get(b);
+ BlockInfo addBlockCollection(BlockInfo b, BlockCollection bc) {
+ BlockInfo info = blocks.get(b);
if (info != b) {
info = b;
blocks.put(info);
@@ -120,7 +120,7 @@ class BlocksMap {
* and remove all data-node locations associated with the block.
*/
void removeBlock(Block block) {
- BlockInfoContiguous blockInfo = blocks.remove(block);
+ BlockInfo blockInfo = blocks.remove(block);
if (blockInfo == null)
return;
@@ -132,7 +132,7 @@ class BlocksMap {
}
/** Returns the block object it it exists in the map. */
- BlockInfoContiguous getStoredBlock(Block b) {
+ BlockInfo getStoredBlock(Block b) {
return blocks.get(b);
}
@@ -164,7 +164,7 @@ class BlocksMap {
* For a block that has already been retrieved from the BlocksMap
* returns {@link Iterable} of the storages the block belongs to.
*/
- Iterable<DatanodeStorageInfo> getStorages(final BlockInfoContiguous storedBlock) {
+ Iterable<DatanodeStorageInfo> getStorages(final BlockInfo storedBlock) {
return new Iterable<DatanodeStorageInfo>() {
@Override
public Iterator<DatanodeStorageInfo> iterator() {
@@ -175,7 +175,7 @@ class BlocksMap {
/** counts number of containing nodes. Better than using iterator. */
int numNodes(Block b) {
- BlockInfoContiguous info = blocks.get(b);
+ BlockInfo info = blocks.get(b);
return info == null ? 0 : info.numNodes();
}
@@ -185,7 +185,7 @@ class BlocksMap {
* only if it does not belong to any file and data-nodes.
*/
boolean removeNode(Block b, DatanodeDescriptor node) {
- BlockInfoContiguous info = blocks.get(b);
+ BlockInfo info = blocks.get(b);
if (info == null)
return false;
@@ -203,7 +203,7 @@ class BlocksMap {
return blocks.size();
}
- Iterable<BlockInfoContiguous> getBlocks() {
+ Iterable<BlockInfo> getBlocks() {
return blocks;
}
@@ -218,8 +218,8 @@ class BlocksMap {
* @param newBlock - block for replacement
* @return new block
*/
- BlockInfoContiguous replaceBlock(BlockInfoContiguous newBlock) {
- BlockInfoContiguous currentBlock = blocks.get(newBlock);
+ BlockInfo replaceBlock(BlockInfo newBlock) {
+ BlockInfo currentBlock = blocks.get(newBlock);
assert currentBlock != null : "the block if not in blocksMap";
// replace block in data-node lists
for (int i = currentBlock.numNodes() - 1; i >= 0; i--) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
index bf5ece9..a0f3503 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
@@ -369,7 +369,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
* @param file The file.
*/
private void rescanFile(CacheDirective directive, INodeFile file) {
- BlockInfoContiguous[] blockInfos = file.getBlocks();
+ BlockInfo[] blockInfos = file.getBlocks();
// Increment the "needed" statistics
directive.addFilesNeeded(1);
@@ -394,7 +394,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
}
long cachedTotal = 0;
- for (BlockInfoContiguous blockInfo : blockInfos) {
+ for (BlockInfo blockInfo : blockInfos) {
if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) {
// We don't try to cache blocks that are under construction.
LOG.trace("Directive {}: can't cache block {} because it is in state "
@@ -453,7 +453,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
}
private String findReasonForNotCaching(CachedBlock cblock,
- BlockInfoContiguous blockInfo) {
+ BlockInfo blockInfo) {
if (blockInfo == null) {
// Somehow, a cache report with the block arrived, but the block
// reports from the DataNode haven't (yet?) described such a block.
@@ -513,7 +513,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
iter.remove();
}
}
- BlockInfoContiguous blockInfo = blockManager.
+ BlockInfo blockInfo = blockManager.
getStoredBlock(new Block(cblock.getBlockId()));
String reason = findReasonForNotCaching(cblock, blockInfo);
int neededCached = 0;
@@ -628,7 +628,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
List<DatanodeDescriptor> pendingCached) {
// To figure out which replicas can be cached, we consult the
// blocksMap. We don't want to try to cache a corrupt replica, though.
- BlockInfoContiguous blockInfo = blockManager.
+ BlockInfo blockInfo = blockManager.
getStoredBlock(new Block(cachedBlock.getBlockId()));
if (blockInfo == null) {
LOG.debug("Block {}: can't add new cached replicas," +
@@ -667,7 +667,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
Iterator<CachedBlock> it = datanode.getPendingCached().iterator();
while (it.hasNext()) {
CachedBlock cBlock = it.next();
- BlockInfoContiguous info =
+ BlockInfo info =
blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
if (info != null) {
pendingBytes -= info.getNumBytes();
@@ -677,7 +677,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
// Add pending uncached blocks from effective capacity
while (it.hasNext()) {
CachedBlock cBlock = it.next();
- BlockInfoContiguous info =
+ BlockInfo info =
blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
if (info != null) {
pendingBytes += info.getNumBytes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 4731ad4..02abc1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -335,7 +335,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
* Remove block from the list of blocks belonging to the data-node. Remove
* data-node from the block.
*/
- boolean removeBlock(BlockInfoContiguous b) {
+ boolean removeBlock(BlockInfo b) {
final DatanodeStorageInfo s = b.findStorageInfo(this);
// if block exists on this datanode
if (s != null) {
@@ -348,7 +348,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
* Remove block from the list of blocks belonging to the data-node. Remove
* data-node from the block.
*/
- boolean removeBlock(String storageID, BlockInfoContiguous b) {
+ boolean removeBlock(String storageID, BlockInfo b) {
DatanodeStorageInfo s = getStorageInfo(storageID);
if (s != null) {
return s.removeBlock(b);
@@ -537,12 +537,12 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
}
- private static class BlockIterator implements Iterator<BlockInfoContiguous> {
+ private static class BlockIterator implements Iterator<BlockInfo> {
private int index = 0;
- private final List<Iterator<BlockInfoContiguous>> iterators;
+ private final List<Iterator<BlockInfo>> iterators;
private BlockIterator(final DatanodeStorageInfo... storages) {
- List<Iterator<BlockInfoContiguous>> iterators = new ArrayList<Iterator<BlockInfoContiguous>>();
+ List<Iterator<BlockInfo>> iterators = new ArrayList<Iterator<BlockInfo>>();
for (DatanodeStorageInfo e : storages) {
iterators.add(e.getBlockIterator());
}
@@ -556,7 +556,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
@Override
- public BlockInfoContiguous next() {
+ public BlockInfo next() {
update();
return iterators.get(index).next();
}
@@ -573,10 +573,10 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
}
- Iterator<BlockInfoContiguous> getBlockIterator() {
+ Iterator<BlockInfo> getBlockIterator() {
return new BlockIterator(getStorageInfos());
}
- Iterator<BlockInfoContiguous> getBlockIterator(final String storageID) {
+ Iterator<BlockInfo> getBlockIterator(final String storageID) {
return new BlockIterator(getStorageInfo(storageID));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index c6c9001..57e81b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -25,9 +25,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
@@ -83,10 +80,10 @@ public class DatanodeStorageInfo {
/**
* Iterates over the list of blocks belonging to the data-node.
*/
- class BlockIterator implements Iterator<BlockInfoContiguous> {
- private BlockInfoContiguous current;
+ class BlockIterator implements Iterator<BlockInfo> {
+ private BlockInfo current;
- BlockIterator(BlockInfoContiguous head) {
+ BlockIterator(BlockInfo head) {
this.current = head;
}
@@ -94,8 +91,8 @@ public class DatanodeStorageInfo {
return current != null;
}
- public BlockInfoContiguous next() {
- BlockInfoContiguous res = current;
+ public BlockInfo next() {
+ BlockInfo res = current;
current = current.getNext(current.findStorageInfo(DatanodeStorageInfo.this));
return res;
}
@@ -115,7 +112,7 @@ public class DatanodeStorageInfo {
private volatile long remaining;
private long blockPoolUsed;
- private volatile BlockInfoContiguous blockList = null;
+ private volatile BlockInfo blockList = null;
private int numBlocks = 0;
// The ID of the last full block report which updated this storage.
@@ -229,7 +226,7 @@ public class DatanodeStorageInfo {
return blockPoolUsed;
}
- public AddBlockResult addBlock(BlockInfoContiguous b) {
+ public AddBlockResult addBlock(BlockInfo b) {
// First check whether the block belongs to a different storage
// on the same DN.
AddBlockResult result = AddBlockResult.ADDED;
@@ -254,7 +251,7 @@ public class DatanodeStorageInfo {
return result;
}
- public boolean removeBlock(BlockInfoContiguous b) {
+ public boolean removeBlock(BlockInfo b) {
blockList = b.listRemove(blockList, this);
if (b.removeStorage(this)) {
numBlocks--;
@@ -268,7 +265,7 @@ public class DatanodeStorageInfo {
return numBlocks;
}
- Iterator<BlockInfoContiguous> getBlockIterator() {
+ Iterator<BlockInfo> getBlockIterator() {
return new BlockIterator(blockList);
}
@@ -277,7 +274,7 @@ public class DatanodeStorageInfo {
* Move block to the head of the list of blocks belonging to the data-node.
* @return the index of the head of the blockList
*/
- int moveBlockToHead(BlockInfoContiguous b, int curIndex, int headIndex) {
+ int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) {
blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex);
return curIndex;
}
@@ -287,7 +284,7 @@ public class DatanodeStorageInfo {
* @return the head of the blockList
*/
@VisibleForTesting
- BlockInfoContiguous getBlockListHeadForTesting(){
+ BlockInfo getBlockListHeadForTesting(){
return blockList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index 5f7366e..639d2b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -100,7 +100,7 @@ public class DecommissionManager {
* reports or other events. Before being finally marking as decommissioned,
* another check is done with the actual block map.
*/
- private final TreeMap<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>
+ private final TreeMap<DatanodeDescriptor, AbstractList<BlockInfo>>
decomNodeBlocks;
/**
@@ -248,7 +248,7 @@ public class DecommissionManager {
* Full-strength replication is not always necessary, hence "sufficient".
* @return true if sufficient, else false.
*/
- private boolean isSufficientlyReplicated(BlockInfoContiguous block,
+ private boolean isSufficientlyReplicated(BlockInfo block,
BlockCollection bc,
NumberReplicas numberReplicas) {
final int numExpected = bc.getPreferredBlockReplication();
@@ -412,7 +412,7 @@ public class DecommissionManager {
}
private void check() {
- final Iterator<Map.Entry<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>>
+ final Iterator<Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>>>
it = new CyclicIteration<>(decomNodeBlocks, iterkey).iterator();
final LinkedList<DatanodeDescriptor> toRemove = new LinkedList<>();
@@ -420,10 +420,10 @@ public class DecommissionManager {
&& !exceededNumBlocksPerCheck()
&& !exceededNumNodesPerCheck()) {
numNodesChecked++;
- final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>
+ final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>>
entry = it.next();
final DatanodeDescriptor dn = entry.getKey();
- AbstractList<BlockInfoContiguous> blocks = entry.getValue();
+ AbstractList<BlockInfo> blocks = entry.getValue();
boolean fullScan = false;
if (blocks == null) {
// This is a newly added datanode, run through its list to schedule
@@ -495,7 +495,7 @@ public class DecommissionManager {
* datanode.
*/
private void pruneSufficientlyReplicated(final DatanodeDescriptor datanode,
- AbstractList<BlockInfoContiguous> blocks) {
+ AbstractList<BlockInfo> blocks) {
processBlocksForDecomInternal(datanode, blocks.iterator(), null, true);
}
@@ -509,9 +509,9 @@ public class DecommissionManager {
* @param datanode
* @return List of insufficiently replicated blocks
*/
- private AbstractList<BlockInfoContiguous> handleInsufficientlyReplicated(
+ private AbstractList<BlockInfo> handleInsufficientlyReplicated(
final DatanodeDescriptor datanode) {
- AbstractList<BlockInfoContiguous> insufficient = new ChunkedArrayList<>();
+ AbstractList<BlockInfo> insufficient = new ChunkedArrayList<>();
processBlocksForDecomInternal(datanode, datanode.getBlockIterator(),
insufficient, false);
return insufficient;
@@ -535,8 +535,8 @@ public class DecommissionManager {
*/
private void processBlocksForDecomInternal(
final DatanodeDescriptor datanode,
- final Iterator<BlockInfoContiguous> it,
- final List<BlockInfoContiguous> insufficientlyReplicated,
+ final Iterator<BlockInfo> it,
+ final List<BlockInfo> insufficientlyReplicated,
boolean pruneSufficientlyReplicated) {
boolean firstReplicationLog = true;
int underReplicatedBlocks = 0;
@@ -544,7 +544,7 @@ public class DecommissionManager {
int underReplicatedInOpenFiles = 0;
while (it.hasNext()) {
numBlocksChecked++;
- final BlockInfoContiguous block = it.next();
+ final BlockInfo block = it.next();
// Remove the block from the list if it's no longer in the block map,
// e.g. the containing file has been deleted
if (blockManager.blocksMap.getStoredBlock(block) == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 307bd59..3b5f973 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
@@ -235,7 +235,7 @@ class FSDirWriteFileOp {
return onRetryBlock[0];
} else {
// add new chosen targets to already allocated block and return
- BlockInfoContiguous lastBlockInFile = pendingFile.getLastBlock();
+ BlockInfo lastBlockInFile = pendingFile.getLastBlock();
((BlockInfoContiguousUnderConstruction) lastBlockInFile)
.setExpectedLocations(targets);
offset = pendingFile.computeFileSize();
@@ -507,7 +507,7 @@ class FSDirWriteFileOp {
/**
* Add a block to the file. Returns a reference to the added block.
*/
- private static BlockInfoContiguous addBlock(
+ private static BlockInfo addBlock(
FSDirectory fsd, String path, INodesInPath inodesInPath, Block block,
DatanodeStorageInfo[] targets) throws IOException {
fsd.writeLock();
@@ -608,7 +608,7 @@ class FSDirWriteFileOp {
}
}
final INodeFile file = fsn.checkLease(src, clientName, inode, fileId);
- BlockInfoContiguous lastBlockInFile = file.getLastBlock();
+ BlockInfo lastBlockInFile = file.getLastBlock();
if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
// The block that the client claims is the current last block
// doesn't match up with what we think is the last block. There are
@@ -636,7 +636,7 @@ class FSDirWriteFileOp {
// changed the namesystem state yet.
// We run this analysis again in Part II where case 4 is impossible.
- BlockInfoContiguous penultimateBlock = file.getPenultimateBlock();
+ BlockInfo penultimateBlock = file.getPenultimateBlock();
if (previous == null &&
lastBlockInFile != null &&
lastBlockInFile.getNumBytes() >= file.getPreferredBlockSize() &&
@@ -762,7 +762,7 @@ class FSDirWriteFileOp {
long id, PermissionStatus permissions, long mtime, long atime,
short replication, long preferredBlockSize, byte storagePolicyId) {
return new INodeFile(id, null, permissions, mtime, atime,
- BlockInfoContiguous.EMPTY_ARRAY, replication, preferredBlockSize,
+ BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize,
storagePolicyId);
}
@@ -802,7 +802,7 @@ class FSDirWriteFileOp {
DatanodeStorageInfo[] targets)
throws IOException {
assert fsn.hasWriteLock();
- BlockInfoContiguous b = addBlock(fsn.dir, src, inodesInPath, newBlock,
+ BlockInfo b = addBlock(fsn.dir, src, inodesInPath, newBlock,
targets);
NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
DatanodeStorageInfo.incrementBlocksScheduled(targets);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 8fdd2d7..5b8650e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -33,10 +33,8 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
@@ -51,7 +49,7 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -919,7 +917,7 @@ public class FSDirectory implements Closeable {
unprotectedTruncate(iip, newLength, collectedBlocks, mtime, null);
if(! onBlockBoundary) {
- BlockInfoContiguous oldBlock = file.getLastBlock();
+ BlockInfo oldBlock = file.getLastBlock();
Block tBlk =
getFSNamesystem().prepareFileForTruncate(iip,
clientName, clientMachine, file.computeFileSize() - newLength,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 83e52bc..dec9709 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.Storage.FormatConfirmable;
@@ -773,10 +773,10 @@ public class FSEditLog implements LogsPurgeable {
public void logAddBlock(String path, INodeFile file) {
Preconditions.checkArgument(file.isUnderConstruction());
- BlockInfoContiguous[] blocks = file.getBlocks();
+ BlockInfo[] blocks = file.getBlocks();
Preconditions.checkState(blocks != null && blocks.length > 0);
- BlockInfoContiguous pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null;
- BlockInfoContiguous lastBlock = blocks[blocks.length - 1];
+ BlockInfo pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null;
+ BlockInfo lastBlock = blocks[blocks.length - 1];
AddBlockOp op = AddBlockOp.getInstance(cache.get()).setPath(path)
.setPenultimateBlock(pBlock).setLastBlock(lastBlock);
logEdit(op);
[02/50] [abbrv] hadoop git commit: HADOOP-9891. CLIMiniCluster
instructions fail with MiniYarnCluster ClassNotFoundException (Darrell Taylor
via aw)
Posted by zj...@apache.org.
HADOOP-9891. CLIMiniCluster instructions fail with MiniYarnCluster ClassNotFoundException (Darrell Taylor via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cd01a19
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cd01a19
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cd01a19
Branch: refs/heads/YARN-2928
Commit: 0cd01a198b8885b63a635bf84b2efc6373b39fa3
Parents: 8b88df7
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed May 27 16:36:55 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 15:46:59 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/site/markdown/CLIMiniCluster.md.vm | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd01a19/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 552443c..d29a885 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -463,6 +463,9 @@ Trunk (Unreleased)
HADOOP-11775. Fix Javadoc typos in hadoop-openstack module (Yanjun Wang
via aw)
+ HADOOP-9891. CLIMiniCluster instructions fail with MiniYarnCluster
+ ClassNotFoundException (Darrell Taylor via aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cd01a19/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
index 25fecda..c7da62b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
@@ -44,7 +44,7 @@ Running the MiniCluster
From inside the root directory of the extracted tarball, you can start the CLI MiniCluster using the following command:
- $ bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${project.version}-tests.jar minicluster -rmport RM_PORT -jhsport JHS_PORT
+ $ HADOOP_CLASSPATH=share/hadoop/yarn/test/hadoop-yarn-server-tests-${project.version}-tests.jar bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${project.version}-tests.jar minicluster -rmport RM_PORT -jhsport JHS_PORT
In the example command above, `RM_PORT` and `JHS_PORT` should be replaced by the user's choice of these port numbers. If not specified, random free ports will be used.
[11/50] [abbrv] hadoop git commit: HADOOP-11406. xargs -P is not
portable (Kengo Seki via aw)
Posted by zj...@apache.org.
HADOOP-11406. xargs -P is not portable (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c65e8ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c65e8ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c65e8ab
Branch: refs/heads/YARN-2928
Commit: 6c65e8ab54c51f234dc4b269cc6decd754c1f86b
Parents: de39fed
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 10:36:40 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../src/main/bin/hadoop-functions.sh | 42 +++++++++++---------
.../main/conf/hadoop-user-functions.sh.example | 29 +++++++++++++-
3 files changed, 54 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c65e8ab/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index eb1db29..7da02ed 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -466,6 +466,8 @@ Trunk (Unreleased)
HADOOP-9891. CLIMiniCluster instructions fail with MiniYarnCluster
ClassNotFoundException (Darrell Taylor via aw)
+ HADOOP-11406. xargs -P is not portable (Kengo Seki via aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c65e8ab/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 67e8870..5556f2f 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -461,27 +461,33 @@ function hadoop_connect_to_hosts
if [[ -z "${SLAVE_NAMES}" ]]; then
SLAVE_NAMES=$(sed 's/#.*$//;/^$/d' "${SLAVE_FILE}")
fi
-
- # quoting here gets tricky. it's easier to push it into a function
- # so that we don't have to deal with it. However...
- # xargs can't use a function so instead we'll export it out
- # and force it into a subshell
- # moral of the story: just use pdsh.
- export -f hadoop_actual_ssh
- export HADOOP_SSH_OPTS
-
- # xargs is used with option -I to replace the placeholder in arguments
- # list with each hostname read from stdin/pipe. But it consider one
- # line as one argument while reading from stdin/pipe. So place each
- # hostname in different lines while passing via pipe.
- SLAVE_NAMES=$(echo "$SLAVE_NAMES" | tr ' ' '\n' )
- echo "${SLAVE_NAMES}" | \
- xargs -n 1 -P"${HADOOP_SSH_PARALLEL}" \
- -I {} bash -c -- "hadoop_actual_ssh {} ${params}"
- wait
+ hadoop_connect_to_hosts_without_pdsh "${params}"
fi
}
+## @description Connect to ${SLAVE_NAMES} and execute command
+## @description under the environment which does not support pdsh.
+## @audience private
+## @stability evolving
+## @replaceable yes
+## @param command
+## @param [...]
+function hadoop_connect_to_hosts_without_pdsh
+{
+ # shellcheck disable=SC2124
+ local params="$@"
+ local slaves=(${SLAVE_NAMES})
+ for (( i = 0; i < ${#slaves[@]}; i++ ))
+ do
+ if (( i != 0 && i % HADOOP_SSH_PARALLEL == 0 )); then
+ wait
+ fi
+ # shellcheck disable=SC2086
+ hadoop_actual_ssh "${slaves[$i]}" ${params} &
+ done
+ wait
+}
+
## @description Utility routine to handle --slaves mode
## @audience private
## @stability evolving
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c65e8ab/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
index b2f78f8..3cf5776 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
@@ -50,7 +50,7 @@
#
#
-# Another example: finding java
+# Example: finding java
#
# By default, Hadoop assumes that $JAVA_HOME is always defined
# outside of its configuration. Eons ago, Apple standardized
@@ -85,3 +85,30 @@
# exit 1
# fi
#}
+
+#
+# Example: efficient command execution for the slaves
+#
+# To improve performance, you can use xargs -P
+# instead of the for loop, if supported.
+#
+#function hadoop_connect_to_hosts_without_pdsh
+#{
+# # quoting here gets tricky. it's easier to push it into a function
+# # so that we don't have to deal with it. However...
+# # xargs can't use a function so instead we'll export it out
+# # and force it into a subshell
+# # moral of the story: just use pdsh.
+# export -f hadoop_actual_ssh
+# export HADOOP_SSH_OPTS
+#
+# # xargs is used with option -I to replace the placeholder in arguments
+# # list with each hostname read from stdin/pipe. But it consider one
+# # line as one argument while reading from stdin/pipe. So place each
+# # hostname in different lines while passing via pipe.
+# SLAVE_NAMES=$(echo "$SLAVE_NAMES" | tr ' ' '\n' )
+# echo "${SLAVE_NAMES}" | \
+# xargs -n 1 -P"${HADOOP_SSH_PARALLEL}" \
+# -I {} bash -c -- "hadoop_actual_ssh {} ${params}"
+# wait
+#}
[39/50] [abbrv] hadoop git commit: YARN-2900. Application (Attempt
and Container) Not Found in AHS results in Internal Server Error (500).
Contributed by Zhijie Shen and Mit Desai
Posted by zj...@apache.org.
YARN-2900. Application (Attempt and Container) Not Found in AHS results
in Internal Server Error (500). Contributed by Zhijie Shen and Mit Desai
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0ecae35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0ecae35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0ecae35
Branch: refs/heads/YARN-2928
Commit: c0ecae35275a96d69c5e5148f0fec91866a14950
Parents: e2101ca
Author: Xuan <xg...@apache.org>
Authored: Sat May 30 21:56:19 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:59 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../timeline/webapp/TimelineWebServices.java | 2 -
.../TestApplicationHistoryClientService.java | 70 +++++++++++++++++++-
.../hadoop/yarn/server/webapp/WebServices.java | 27 +++++---
4 files changed, 89 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ecae35/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6f5faa9..45c4f68 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -694,6 +694,9 @@ Release 2.7.1 - UNRELEASED
YARN-3686. CapacityScheduler should trim default_node_label_expression.
(Sunil G via wangda)
+ YARN-2900. Application (Attempt and Container) Not Found in AHS results
+ in InternalServer Error (500). (Zhijie Shen and Mit Desai via xgong)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ecae35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
index 90f4d39..40e9fed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
@@ -47,7 +47,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@@ -61,7 +60,6 @@ import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
-import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ecae35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 764ecb9..d04903c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
@@ -54,12 +57,13 @@ public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService;
private static TimelineDataManager dataManager;
+ private final static int MAX_APPS = 2;
@BeforeClass
public static void setup() throws Exception {
Configuration conf = new YarnConfiguration();
TimelineStore store =
- TestApplicationHistoryManagerOnTimelineStore.createStore(2);
+ TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
dataManager =
new TimelineDataManager(store, aclsManager);
@@ -72,6 +76,70 @@ public class TestApplicationHistoryClientService {
}
@Test
+ public void testApplicationNotFound() throws IOException, YarnException {
+ ApplicationId appId = null;
+ appId = ApplicationId.newInstance(0, MAX_APPS + 1);
+ GetApplicationReportRequest request =
+ GetApplicationReportRequest.newInstance(appId);
+ try {
+ @SuppressWarnings("unused")
+ GetApplicationReportResponse response =
+ clientService.getApplicationReport(request);
+ Assert.fail("Exception should have been thrown before we reach here.");
+ } catch (ApplicationNotFoundException e) {
+ //This exception is expected.
+ Assert.assertTrue(e.getMessage().contains(
+ "doesn't exist in the timeline store"));
+ } catch (Exception e) {
+ Assert.fail("Undesired exception caught");
+ }
+ }
+
+ @Test
+ public void testApplicationAttemptNotFound() throws IOException, YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
+ GetApplicationAttemptReportRequest request =
+ GetApplicationAttemptReportRequest.newInstance(appAttemptId);
+ try {
+ @SuppressWarnings("unused")
+ GetApplicationAttemptReportResponse response =
+ clientService.getApplicationAttemptReport(request);
+ Assert.fail("Exception should have been thrown before we reach here.");
+ } catch (ApplicationAttemptNotFoundException e) {
+ //This Exception is expected
+ System.out.println(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains(
+ "doesn't exist in the timeline store"));
+ } catch (Exception e) {
+ Assert.fail("Undesired exception caught");
+ }
+ }
+
+ @Test
+ public void testContainerNotFound() throws IOException, YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId = ContainerId.newContainerId(appAttemptId,
+ MAX_APPS + 1);
+ GetContainerReportRequest request =
+ GetContainerReportRequest.newInstance(containerId);
+ try {
+ @SuppressWarnings("unused")
+ GetContainerReportResponse response =
+ clientService.getContainerReport(request);
+ } catch (ContainerNotFoundException e) {
+ //This exception is expected
+ Assert.assertTrue(e.getMessage().contains(
+ "doesn't exist in the timeline store"));
+ } catch (Exception e) {
+ Assert.fail("Undesired exception caught");
+ }
+ }
+
+ @Test
public void testApplicationReport() throws IOException, YarnException {
ApplicationId appId = null;
appId = ApplicationId.newInstance(0, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ecae35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index 8404719..ae27d45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -47,6 +47,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@@ -477,17 +480,21 @@ public class WebServices {
private static void rewrapAndThrowException(Exception e) {
if (e instanceof UndeclaredThrowableException) {
- if (e.getCause() instanceof AuthorizationException) {
- throw new ForbiddenException(e.getCause());
- } else {
- throw new WebApplicationException(e.getCause());
- }
+ rewrapAndThrowThrowable(e.getCause());
} else {
- if (e instanceof AuthorizationException) {
- throw new ForbiddenException(e);
- } else {
- throw new WebApplicationException(e);
- }
+ rewrapAndThrowThrowable(e);
+ }
+ }
+
+ private static void rewrapAndThrowThrowable(Throwable t) {
+ if (t instanceof AuthorizationException) {
+ throw new ForbiddenException(t);
+ } if (t instanceof ApplicationNotFoundException ||
+ t instanceof ApplicationAttemptNotFoundException ||
+ t instanceof ContainerNotFoundException) {
+ throw new NotFoundException(t);
+ } else {
+ throw new WebApplicationException(t);
}
}
[38/50] [abbrv] hadoop git commit: YARN-3467. Expose allocatedMB,
allocatedVCores,
and runningContainers metrics on running Applications in RM Web UI. (Anubhav
Dhoot via kasha)
Posted by zj...@apache.org.
YARN-3467. Expose allocatedMB, allocatedVCores, and runningContainers metrics on running Applications in RM Web UI. (Anubhav Dhoot via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af69d9c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af69d9c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af69d9c1
Branch: refs/heads/YARN-2928
Commit: af69d9c14484d9ece37d2ed505ea23ec63a891f9
Parents: d0c491b
Author: Karthik Kambatla <ka...@apache.org>
Authored: Sat May 30 13:36:24 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:59 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/webapp/WebPageUtils.java | 4 ++--
.../apache/hadoop/yarn/server/webapp/dao/AppInfo.java | 14 ++++++++++++++
.../webapp/FairSchedulerAppsBlock.java | 6 ++++++
.../server/resourcemanager/webapp/RMAppsBlock.java | 8 ++++++++
5 files changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 339203c..2c48701 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -371,6 +371,9 @@ Release 2.8.0 - UNRELEASED
YARN-3713. Remove duplicate function call storeContainerDiagnostics in
ContainerDiagnosticsUpdateTransition (zxu via rkanter)
+ YARN-3467. Expose allocatedMB, allocatedVCores, and runningContainers metrics on
+ running Applications in RM Web UI. (Anubhav Dhoot via kasha)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
index 6ca5011..ed0fe38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
@@ -52,9 +52,9 @@ public class WebPageUtils {
.append(", 'mRender': renderHadoopDate }")
.append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':");
if (isFairSchedulerPage) {
- sb.append("[11]");
+ sb.append("[13]");
} else if (isResourceManager) {
- sb.append("[10]");
+ sb.append("[12]");
} else {
sb.append("[9]");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index 5d9ba6f..0cc5f75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -58,6 +58,8 @@ public class AppInfo {
protected long finishedTime;
protected long elapsedTime;
protected String applicationTags;
+ private int allocatedCpuVcores;
+ private int allocatedMemoryMB;
public AppInfo() {
// JAXB needs this
@@ -86,6 +88,10 @@ public class AppInfo {
if (app.getApplicationResourceUsageReport() != null) {
runningContainers =
app.getApplicationResourceUsageReport().getNumUsedContainers();
+ allocatedCpuVcores = app.getApplicationResourceUsageReport()
+ .getUsedResources().getVirtualCores();
+ allocatedMemoryMB = app.getApplicationResourceUsageReport()
+ .getUsedResources().getMemory();
}
progress = app.getProgress() * 100; // in percent
if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
@@ -133,6 +139,14 @@ public class AppInfo {
return runningContainers;
}
+ public int getAllocatedCpuVcores() {
+ return allocatedCpuVcores;
+ }
+
+ public int getAllocatedMemoryMB() {
+ return allocatedMemoryMB;
+ }
+
public float getProgress() {
return progress;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index 80099d0..f257656 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -93,6 +93,8 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
th(".state", "State").
th(".finalstatus", "FinalStatus").
th(".runningcontainer", "Running Containers").
+ th(".allocatedCpu", "Allocated CPU VCores").
+ th(".allocatedMemory", "Allocated Memory MB").
th(".progress", "Progress").
th(".ui", "Tracking UI")._()._().
tbody();
@@ -136,6 +138,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
.append(appInfo.getFinalStatus()).append("\",\"")
.append(appInfo.getRunningContainers() == -1 ? "N/A" : String
.valueOf(appInfo.getRunningContainers())).append("\",\"")
+ .append(appInfo.getAllocatedVCores() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedVCores())).append("\",\"")
+ .append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedMB())).append("\",\"")
// Progress bar
.append("<br title='").append(percent)
.append("'> <div class='").append(C_PROGRESSBAR).append("' title='")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af69d9c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index 1d97bef..d252c30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -60,6 +60,8 @@ public class RMAppsBlock extends AppsBlock {
.th(".finishtime", "FinishTime").th(".state", "State")
.th(".finalstatus", "FinalStatus")
.th(".runningcontainer", "Running Containers")
+ .th(".allocatedCpu", "Allocated CPU VCores")
+ .th(".allocatedMemory", "Allocated Memory MB")
.th(".progress", "Progress")
.th(".ui", "Tracking UI").th(".blacklisted", "Blacklisted Nodes")._()
._().tbody();
@@ -114,6 +116,12 @@ public class RMAppsBlock extends AppsBlock {
.append(app.getRunningContainers() == -1 ? "N/A" : String
.valueOf(app.getRunningContainers()))
.append("\",\"")
+ .append(app.getAllocatedCpuVcores() == -1 ? "N/A" : String
+ .valueOf(app.getAllocatedCpuVcores()))
+ .append("\",\"")
+ .append(app.getAllocatedMemoryMB() == -1 ? "N/A" : String
+ .valueOf(app.getAllocatedMemoryMB()))
+ .append("\",\"")
// Progress bar
.append("<br title='").append(percent).append("'> <div class='")
.append(C_PROGRESSBAR).append("' title='").append(join(percent, '%'))
[27/50] [abbrv] hadoop git commit: HADOOP-12022. fix site -Pdocs
-Pdist in hadoop-project-dist; cleanout remaining forrest bits (aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
deleted file mode 100644
index 9880272..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
+++ /dev/null
@@ -1,29099 +0,0 @@
-<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<title>Hadoop 2.6.0 Release Notes</title>
-<STYLE type="text/css">
- H1 {font-family: sans-serif}
- H2 {font-family: sans-serif; margin-left: 7mm}
- TABLE {margin-left: 7mm}
-</STYLE>
-</head>
-<body>
-<h1>Hadoop 2.6.0 Release Notes</h1>
-These release notes include new developer and user-facing incompatibilities, features, and major improvements.
-<a name="changes"/>
-<h2>Changes since Hadoop 2.5.1</h2>
-<ul>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2853">YARN-2853</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>Killing app may hang while AM is unregistering</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2846">YARN-2846</a>.
- Blocker bug reported by Junping Du and fixed by Junping Du (nodemanager)<br>
- <b>Incorrect persist exit code for running containers in reacquireContainer() that interrupted by NodeManager restart.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2843">YARN-2843</a>.
- Major sub-task reported by Sushmitha Sreenivasan and fixed by Wangda Tan (resourcemanager)<br>
- <b>NodeLabels manager should trim all inputs for hosts and labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2841">YARN-2841</a>.
- Critical sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>RMProxy should retry EOFException </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2834">YARN-2834</a>.
- Blocker bug reported by Yesha Vora and fixed by Jian He <br>
- <b>Resource manager crashed with Null Pointer Exception</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2830">YARN-2830</a>.
- Blocker bug reported by Jonathan Eagles and fixed by Jonathan Eagles <br>
- <b>Add backwords compatible ContainerId.newInstance constructor for use within Tez Local Mode</b><br>
- <blockquote>I just committed this. Thanks [~jeagles] for the patch and [~ozawa] for the reviews!</blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2827">YARN-2827</a>.
- Critical bug reported by Wangda Tan and fixed by Wangda Tan (client)<br>
- <b>Fix bugs of yarn queue CLI</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2826">YARN-2826</a>.
- Critical bug reported by Sidharta Seethana and fixed by Wangda Tan <br>
- <b>User-Group mappings not updated by RM when a user is removed from a group.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2825">YARN-2825</a>.
- Critical bug reported by Jian He and fixed by Jian He <br>
- <b>Container leak on NM</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2824">YARN-2824</a>.
- Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Capacity of labels should be zero by default</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2823">YARN-2823</a>.
- Critical bug reported by Gour Saha and fixed by Jian He (resourcemanager)<br>
- <b>NullPointerException in RM HA enabled 3-node cluster</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2819">YARN-2819</a>.
- Critical bug reported by Gopal V and fixed by Zhijie Shen (timelineserver)<br>
- <b>NPE in ATS Timeline Domains when upgrading from 2.4 to 2.6</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2818">YARN-2818</a>.
- Critical bug reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Remove the logic to inject entity owner as the primary filter</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2813">YARN-2813</a>.
- Major bug reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>NPE from MemoryTimelineStore.getDomains</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2812">YARN-2812</a>.
- Major test reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>TestApplicationHistoryServer is likely to fail on less powerful machine</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2810">YARN-2810</a>.
- Major test reported by Varun Vasudev and fixed by Varun Vasudev (resourcemanager)<br>
- <b>TestRMProxyUsersConf fails on Windows VMs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2805">YARN-2805</a>.
- Blocker bug reported by Arpit Gupta and fixed by Wangda Tan (resourcemanager)<br>
- <b>RM2 in HA setup tries to login using the RM1's kerberos principal</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2804">YARN-2804</a>.
- Critical bug reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>Timeline server .out log have JAXB binding exceptions and warnings.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2803">YARN-2803</a>.
- Critical bug reported by Chris Nauroth and fixed by Craig Welch (nodemanager)<br>
- <b>MR distributed cache not working correctly on Windows after NodeManager privileged account changes.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2798">YARN-2798</a>.
- Blocker bug reported by Arpit Gupta and fixed by Zhijie Shen (timelineserver)<br>
- <b>YarnClient doesn't need to translate Kerberos name of timeline DT renewer</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2795">YARN-2795</a>.
- Major sub-task reported by Phil D'Amore and fixed by Wangda Tan (resourcemanager)<br>
- <b>Resource Manager fails startup with HDFS label storage and secure cluster</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2794">YARN-2794</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>Fix log msgs about distributing system-credentials </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2790">YARN-2790</a>.
- Critical bug reported by Tassapol Athiapinya and fixed by Jian He (nodemanager)<br>
- <b>NM can't aggregate logs past HDFS delegation token expiry.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2789">YARN-2789</a>.
- Critical task reported by Siddharth Seth and fixed by Wangda Tan <br>
- <b>Re-instate the NodeReport.newInstance API modified in YARN-2698</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2788">YARN-2788</a>.
- Blocker bug reported by Gopal V and fixed by Xuan Gong (log-aggregation)<br>
- <b>yarn logs -applicationId on 2.6.0 should support logs written by 2.4.0</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2785">YARN-2785</a>.
- Major test reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>TestContainerResourceUsage fails intermittently</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2779">YARN-2779</a>.
- Critical bug reported by Zhijie Shen and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
- <b>SystemMetricsPublisher can use Kerberos directly instead of timeline DT</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2778">YARN-2778</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (client)<br>
- <b>YARN node CLI should display labels on returned node reports</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2770">YARN-2770</a>.
- Critical sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Timeline delegation tokens need to be automatically renewed by the RM</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2769">YARN-2769</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev (applications/distributed-shell)<br>
- <b>Timeline server domain not set correctly when using shell_command on Windows</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2767">YARN-2767</a>.
- Major test reported by Varun Vasudev and fixed by Varun Vasudev (resourcemanager)<br>
- <b>RM web services - add test case to ensure the http static user cannot kill or submit apps in secure mode</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2760">YARN-2760</a>.
- Trivial bug reported by Harsh J and fixed by Harsh J (documentation)<br>
- <b>Completely remove word 'experimental' from FairScheduler docs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2758">YARN-2758</a>.
- Major test reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Update TestApplicationHistoryClientService to use the new generic history store</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2755">YARN-2755</a>.
- Critical bug reported by Siqi Li and fixed by Siqi Li <br>
- <b>NM fails to clean up usercache_DEL_<timestamp> dirs after YARN-661</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2753">YARN-2753</a>.
- Major sub-task reported by zhihai xu and fixed by zhihai xu <br>
- <b>Fix potential issues and code clean up for *NodeLabelsManager</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2752">YARN-2752</a>.
- Critical bug reported by Xuan Gong and fixed by Xuan Gong <br>
- <b>ContainerExecutor always append "nice -n" in command on branch-2</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2747">YARN-2747</a>.
- Major test reported by Xuan Gong and fixed by Xuan Gong <br>
- <b>TestAggregatedLogFormat fails in trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2744">YARN-2744</a>.
- Critical sub-task reported by Sumit Mohanty and fixed by Wangda Tan (capacityscheduler)<br>
- <b>Under some scenario, it is possible to end up with capacity scheduler configuration that uses labels that no longer exist</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2743">YARN-2743</a>.
- Blocker bug reported by Arpit Gupta and fixed by Jian He (resourcemanager)<br>
- <b>Yarn jobs via oozie fail with failed to renew token (secure) or digest mismatch (unsecure) errors when RM is being killed</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2741">YARN-2741</a>.
- Major bug reported by Craig Welch and fixed by Craig Welch (nodemanager)<br>
- <b>Windows: Node manager cannot serve up log files via the web user interface when yarn.nodemanager.log-dirs to any drive letter other than C: (or, the drive that nodemanager is running on)</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2734">YARN-2734</a>.
- Major bug reported by Sumit Mohanty and fixed by Xuan Gong (log-aggregation)<br>
- <b>If a sub-folder is encountered by log aggregator it results in invalid aggregated file</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2732">YARN-2732</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>Fix syntax error in SecureContainer.apt.vm</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2730">YARN-2730</a>.
- Critical bug reported by Siqi Li and fixed by Siqi Li <br>
- <b>DefaultContainerExecutor runs only one localizer at a time</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2726">YARN-2726</a>.
- Minor sub-task reported by Phil D'Amore and fixed by Wangda Tan (capacityscheduler)<br>
- <b>CapacityScheduler should explicitly log when an accessible label has no capacity</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2724">YARN-2724</a>.
- Major bug reported by Sumit Mohanty and fixed by Xuan Gong (log-aggregation)<br>
- <b>If an unreadable file is encountered during log aggregation then aggregated file in HDFS badly formed</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2723">YARN-2723</a>.
- Major sub-task reported by Phil D'Amore and fixed by Naganarasimha G R (client)<br>
- <b>rmadmin -replaceLabelsOnNode does not correctly parse port</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2721">YARN-2721</a>.
- Blocker bug reported by Jian He and fixed by Jian He <br>
- <b>Race condition: ZKRMStateStore retry logic may throw NodeExist exception </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2720">YARN-2720</a>.
- Major bug reported by Craig Welch and fixed by Craig Welch (nodemanager)<br>
- <b>Windows: Wildcard classpath variables not expanded against resources contained in archives</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2717">YARN-2717</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (log-aggregation)<br>
- <b>containerLogNotFound log shows multiple time for the same container</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2715">YARN-2715</a>.
- Blocker bug reported by Zhijie Shen and fixed by Zhijie Shen (resourcemanager)<br>
- <b>Proxy user is problem for RPC interface if yarn.resourcemanager.webapp.proxyuser is not set.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2711">YARN-2711</a>.
- Major test reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>TestDefaultContainerExecutor#testContainerLaunchError fails on Windows</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2709">YARN-2709</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu <br>
- <b>Add retry for timeline client getDelegationToken method</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2707">YARN-2707</a>.
- Minor bug reported by Ted Yu and fixed by Gera Shegalov <br>
- <b>Potential null dereference in FSDownload</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2705">YARN-2705</a>.
- Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
- <b>Changes of RM node label manager default configuration</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2704">YARN-2704</a>.
- Critical sub-task reported by Jian He and fixed by Jian He <br>
- <b> Localization and log-aggregation will fail if hdfs delegation token expired after token-max-life-time</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2703">YARN-2703</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>Add logUploadedTime into LogValue for better display</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2701">YARN-2701</a>.
- Blocker bug reported by Xuan Gong and fixed by Xuan Gong <br>
- <b>Potential race condition in startLocalizer when using LinuxContainerExecutor </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2700">YARN-2700</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api , resourcemanager)<br>
- <b>TestSecureRMRegistryOperations failing on windows: auth problems</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2699">YARN-2699</a>.
- Blocker sub-task reported by Wangda Tan and fixed by Wangda Tan (client)<br>
- <b>Fix test timeout in TestResourceTrackerOnHA#testResourceTrackerOnHA</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2698">YARN-2698</a>.
- Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
- <b>Move getClusterNodeLabels and getNodeToLabels to YarnClient instead of AdminService</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2692">YARN-2692</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran <br>
- <b>ktutil test hanging on some machines/ktutil versions</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2689">YARN-2689</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api , resourcemanager)<br>
- <b>TestSecureRMRegistryOperations failing on windows: secure ZK won't start</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2685">YARN-2685</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
- <b>Resource on each label not correct when multiple NMs in a same host and some has label some not</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2682">YARN-2682</a>.
- Minor bug reported by zhihai xu and fixed by zhihai xu (nodemanager)<br>
- <b>WindowsSecureContainerExecutor should not depend on DefaultContainerExecutor#getFirstApplicationDir. </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2678">YARN-2678</a>.
- Major sub-task reported by Gour Saha and fixed by Steve Loughran (api , resourcemanager)<br>
- <b>Improved Yarn Registry service record structure</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2677">YARN-2677</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api , resourcemanager)<br>
- <b>registry punycoding of usernames doesn't fix all usernames to be DNS-valid</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2676">YARN-2676</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Timeline authentication filter should add support for proxy user</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2673">YARN-2673</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu <br>
- <b>Add retry for timeline client put APIs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2671">YARN-2671</a>.
- Blocker bug reported by Zhijie Shen and fixed by Wangda Tan (resourcemanager)<br>
- <b>ApplicationSubmissionContext change breaks the existing app submission</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2668">YARN-2668</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (client)<br>
- <b>yarn-registry JAR won't link against ZK 3.4.5</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2667">YARN-2667</a>.
- Minor bug reported by Yi Liu and fixed by Yi Liu <br>
- <b>Fix the release audit warning caused by hadoop-yarn-registry</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2662">YARN-2662</a>.
- Minor bug reported by Chris Nauroth and fixed by Chris Nauroth (test)<br>
- <b>TestCgroupsLCEResourcesHandler leaks file descriptors.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2656">YARN-2656</a>.
- Major bug reported by Varun Vasudev and fixed by Zhijie Shen (resourcemanager)<br>
- <b>RM web services authentication filter should add support for proxy user</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2652">YARN-2652</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api)<br>
- <b>add hadoop-yarn-registry package under hadoop-yarn</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2651">YARN-2651</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>Spin off the LogRollingInterval from LogAggregationContext</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2649">YARN-2649</a>.
- Major bug reported by Ming Ma and fixed by Ming Ma <br>
- <b>Flaky test TestAMRMRPCNodeUpdates</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2647">YARN-2647</a>.
- Major sub-task reported by Wangda Tan and fixed by Sunil G (client)<br>
- <b>Add yarn queue CLI to get queue infos</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2644">YARN-2644</a>.
- Major sub-task reported by Craig Welch and fixed by Craig Welch <br>
- <b>Recalculate headroom more frequently to keep it accurate</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2635">YARN-2635</a>.
- Major bug reported by Wei Yan and fixed by Wei Yan <br>
- <b>TestRM, TestRMRestart, TestClientToAMTokens should run with both CS and FS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2632">YARN-2632</a>.
- Blocker sub-task reported by Junping Du and fixed by Junping Du (nodemanager)<br>
- <b>Document NM Restart feature</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2630">YARN-2630</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>TestDistributedShell#testDSRestartWithPreviousRunningContainers fails</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2629">YARN-2629</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Make distributed shell use the domain-based timeline ACLs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2628">YARN-2628</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev (capacityscheduler)<br>
- <b>Capacity scheduler with DominantResourceCalculator carries out reservation even though slots are free</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2627">YARN-2627</a>.
- Major improvement reported by Xuan Gong and fixed by Xuan Gong <br>
- <b>Add logs when attemptFailuresValidityInterval is enabled</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2624">YARN-2624</a>.
- Blocker bug reported by Anubhav Dhoot and fixed by Anubhav Dhoot (nodemanager)<br>
- <b>Resource Localization fails on a cluster due to existing cache directories</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2621">YARN-2621</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Simplify the output when the user doesn't have the access for getDomain(s) </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2617">YARN-2617</a>.
- Major bug reported by Jun Gong and fixed by Jun Gong (nodemanager)<br>
- <b>NM does not need to send finished container whose APP is not running to RM</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2615">YARN-2615</a>.
- Blocker sub-task reported by Junping Du and fixed by Junping Du <br>
- <b>ClientToAMTokenIdentifier and DelegationTokenIdentifier should allow extended fields</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2611">YARN-2611</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler , resourcemanager , scheduler)<br>
- <b>Fix jenkins findbugs warning and test case failures for trunk merge patch</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2610">YARN-2610</a>.
- Major bug reported by Ray Chiang and fixed by Ray Chiang <br>
- <b>Hamlet should close table tags</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2608">YARN-2608</a>.
- Major bug reported by Wei Yan and fixed by Wei Yan <br>
- <b>FairScheduler: Potential deadlocks in loading alloc files and clock access</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2607">YARN-2607</a>.
- Major test reported by Ted Yu and fixed by Wangda Tan <br>
- <b>TestDistributedShell fails in trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2606">YARN-2606</a>.
- Major bug reported by Mit Desai and fixed by Mit Desai (timelineserver)<br>
- <b>Application History Server tries to access hdfs before doing secure login</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2602">YARN-2602</a>.
- Major bug reported by Karam Singh and fixed by Zhijie Shen (timelineserver)<br>
- <b>Generic History Service of TimelineServer sometimes not able to handle NPE</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2596">YARN-2596</a>.
- Major test reported by Junping Du and fixed by Karthik Kambatla <br>
- <b>TestWorkPreservingRMRestart fails with FairScheduler</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2594">YARN-2594</a>.
- Blocker bug reported by Karam Singh and fixed by Wangda Tan (resourcemanager)<br>
- <b>Potential deadlock in RM when querying ApplicationResourceUsageReport</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2591">YARN-2591</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>AHSWebServices should return FORBIDDEN(403) if the request user doesn't have access to the history data</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2588">YARN-2588</a>.
- Major bug reported by Rohith and fixed by Rohith (resourcemanager)<br>
- <b>Standby RM does not transitionToActive if previous transitionToActive is failed with ZK exception.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2584">YARN-2584</a>.
- Major test reported by Zhijie Shen and fixed by Jian He <br>
- <b>TestContainerManagerSecurity fails on trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2583">YARN-2583</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>Modify the LogDeletionService to support Log aggregation for LRS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2582">YARN-2582</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>Log related CLI and Web UI changes for Aggregated Logs in LRS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2581">YARN-2581</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>NMs need to find a way to get LogAggregationContext</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2579">YARN-2579</a>.
- Blocker bug reported by Rohith and fixed by Rohith (resourcemanager)<br>
- <b>Deadlock when EmbeddedElectorService and FatalEventDispatcher try to transition RM to StandBy at the same time</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2577">YARN-2577</a>.
- Trivial improvement reported by Miklos Christine and fixed by Miklos Christine (documentation , fairscheduler)<br>
- <b>Clarify ACL delimiter and how to configure ACL groups only</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2576">YARN-2576</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler , resourcemanager , scheduler)<br>
- <b>Prepare yarn-1051 branch for merging with trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2569">YARN-2569</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>API changes for handling logs of long-running services</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2568">YARN-2568</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>TestAMRMClientOnRMRestart test fails</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2566">YARN-2566</a>.
- Critical sub-task reported by zhihai xu and fixed by zhihai xu (nodemanager)<br>
- <b>DefaultContainerExecutor should pick a working directory randomly</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2565">YARN-2565</a>.
- Major bug reported by Karam Singh and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
- <b>RM shouldn't use the old RMApplicationHistoryWriter unless explicitly setting FileSystemApplicationHistoryStore</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2563">YARN-2563</a>.
- Blocker bug reported by Arpit Gupta and fixed by Zhijie Shen (timelineserver)<br>
- <b>On secure clusters call to timeline server fails with authentication errors when running a job via oozie</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2562">YARN-2562</a>.
- Critical bug reported by Vinod Kumar Vavilapalli and fixed by Tsuyoshi OZAWA <br>
- <b>ContainerId@toString() is unreadable for epoch >0 after YARN-2182</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2561">YARN-2561</a>.
- Blocker sub-task reported by Tassapol Athiapinya and fixed by Junping Du <br>
- <b>MR job client cannot reconnect to AM after NM restart.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2559">YARN-2559</a>.
- Major bug reported by Karam Singh and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
- <b>ResourceManager sometime become un-responsive due to NPE in SystemMetricsPublisher</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2558">YARN-2558</a>.
- Blocker sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
- <b>Updating ContainerTokenIdentifier#read/write to use ContainerId#getContainerId</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2557">YARN-2557</a>.
- Major bug reported by Xuan Gong and fixed by Xuan Gong (applications/distributed-shell)<br>
- <b>Add a parameter "attempt_Failures_Validity_Interval" in DistributedShell </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2549">YARN-2549</a>.
- Minor test reported by Chris Nauroth and fixed by Chris Nauroth (nodemanager , test)<br>
- <b>TestContainerLaunch fails due to classpath problem with hamcrest classes.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2547">YARN-2547</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
- <b>Cross Origin Filter throws UnsupportedOperationException upon destroy</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2546">YARN-2546</a>.
- Major bug reported by Doug Haigh and fixed by Varun Vasudev (api)<br>
- <b>REST API for application creation/submission is using strings for numeric & boolean values</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2544">YARN-2544</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
- <b>Common server side PB changes (not include user API PB changes)</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2542">YARN-2542</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>"yarn application -status <appId>" throws NPE when retrieving the app from the timelineserver</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2541">YARN-2541</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>Fix ResourceManagerRest.apt.vm syntax error</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2540">YARN-2540</a>.
- Major bug reported by Ashwin Shankar and fixed by Ashwin Shankar (scheduler)<br>
- <b>FairScheduler: Queue filters not working on scheduler page in RM UI</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2539">YARN-2539</a>.
- Minor improvement reported by Wei Yan and fixed by Wei Yan <br>
- <b>FairScheduler: Set the default value for maxAMShare to 0.5</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2538">YARN-2538</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>Add logs when RM send new AMRMToken to ApplicationMaster</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2534">YARN-2534</a>.
- Major bug reported by zhihai xu and fixed by zhihai xu (scheduler)<br>
- <b>FairScheduler: Potential integer overflow calculating totalMaxShare</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2531">YARN-2531</a>.
- Major improvement reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>CGroups - Admins should be allowed to enforce strict cpu limits</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2529">YARN-2529</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Generic history service RPC interface doesn't work when service authorization is enabled</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2528">YARN-2528</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
- <b>Cross Origin Filter Http response split vulnerability protection rejects valid origins</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2527">YARN-2527</a>.
- Major bug reported by Benoy Antony and fixed by Benoy Antony (resourcemanager)<br>
- <b>NPE in ApplicationACLsManager</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2526">YARN-2526</a>.
- Critical bug reported by Wei Yan and fixed by Wei Yan (scheduler-load-simulator)<br>
- <b>SLS can deadlock when all the threads are taken by AMSimulators</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2523">YARN-2523</a>.
- Major bug reported by Nishan Shetty and fixed by Rohith (resourcemanager , webapp)<br>
- <b>ResourceManager UI showing negative value for "Decommissioned Nodes" field</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2519">YARN-2519</a>.
- Major test reported by Xiaoyu Yao and fixed by Xiaoyu Yao (webapp)<br>
- <b>Credential Provider related unit tests failed on Windows</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2515">YARN-2515</a>.
- Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>Update ConverterUtils#toContainerId to parse epoch</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2512">YARN-2512</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
- <b>Allow for origin pattern matching in cross origin filter</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2511">YARN-2511</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
- <b>Allow All Origins by default when Cross Origin Filter is enabled</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2509">YARN-2509</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
- <b>Enable Cross Origin Filter for timeline server only and not all Yarn servers</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2508">YARN-2508</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
- <b>Cross Origin configuration parameters prefix are not honored</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2507">YARN-2507</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (documentation , timelineserver)<br>
- <b>Document Cross Origin Filter Configuration for ATS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2505">YARN-2505</a>.
- Major sub-task reported by Wangda Tan and fixed by Craig Welch (resourcemanager)<br>
- <b>Support get/add/remove/change labels in RM REST API</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2504">YARN-2504</a>.
- Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Support get/add/remove/change labels in RM admin CLI </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2503">YARN-2503</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Changes in RM Web UI to better show labels to end users</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2502">YARN-2502</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Changes in distributed shell to support specify labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2501">YARN-2501</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Changes in AMRMClient to support labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2500">YARN-2500</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Miscellaneous changes in ResourceManager to support labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2496">YARN-2496</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Changes for capacity scheduler to support allocate resource respect labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2494">YARN-2494</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Node label manager API and storage implementations</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2493">YARN-2493</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api)<br>
- <b>API changes for users</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2484">YARN-2484</a>.
- Trivial bug reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
- <b>FileSystemRMStateStore#readFile/writeFile should close FSData(In|Out)putStream in final block</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2475">YARN-2475</a>.
- Major sub-task reported by Carlo Curino and fixed by Carlo Curino (resourcemanager)<br>
- <b>ReservationSystem: replan upon capacity reduction</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2468">YARN-2468</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (log-aggregation , nodemanager , resourcemanager)<br>
- <b>Log handling for LRS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2462">YARN-2462</a>.
- Major bug reported by Jason Lowe and fixed by Eric Payne <br>
- <b>TestNodeManagerResync#testBlockNewContainerRequestsOnStartAndResync should have a test timeout</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2460">YARN-2460</a>.
- Minor bug reported by Ray Chiang and fixed by Ray Chiang <br>
- <b>Remove obsolete entries from yarn-default.xml</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2459">YARN-2459</a>.
- Major bug reported by Mayank Bansal and fixed by Mayank Bansal (resourcemanager)<br>
- <b>RM crashes if App gets rejected for any reason and HA is enabled</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2456">YARN-2456</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>Possible livelock in CapacityScheduler when RM is recovering apps</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2453">YARN-2453</a>.
- Major bug reported by zhihai xu and fixed by zhihai xu <br>
- <b>TestProportionalCapacityPreemptionPolicy fails with FairScheduler</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2452">YARN-2452</a>.
- Major bug reported by zhihai xu and fixed by zhihai xu <br>
- <b>TestRMApplicationHistoryWriter fails with FairScheduler</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2450">YARN-2450</a>.
- Trivial bug reported by Ray Chiang and fixed by Ray Chiang <br>
- <b>Fix typos in log messages</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2449">YARN-2449</a>.
- Critical bug reported by Karam Singh and fixed by Varun Vasudev (timelineserver)<br>
- <b>Timelineserver returns invalid Delegation token in secure kerberos enabled cluster when hadoop.http.filter.initializers are not set</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2448">YARN-2448</a>.
- Major improvement reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>RM should expose the resource types considered during scheduling when AMs register</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2447">YARN-2447</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>RM web services app submission doesn't pass secrets correctly</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2446">YARN-2446</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Using TimelineNamespace to shield the entities of a user</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2440">YARN-2440</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>Cgroups should allow YARN containers to be limited to allocated cores</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2434">YARN-2434</a>.
- Major sub-task reported by Jian He and fixed by Jian He <br>
- <b>RM should not recover containers from previously failed attempt when AM restart is not enabled</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2431">YARN-2431</a>.
- Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
- <b>NM restart: cgroup is not removed for reacquired containers</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2424">YARN-2424</a>.
- Blocker bug reported by Allen Wittenauer and fixed by Allen Wittenauer (nodemanager)<br>
- <b>LCE should support non-cgroups, non-secure mode</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2411">YARN-2411</a>.
- Major improvement reported by Ram Venkatesh and fixed by Ram Venkatesh (capacityscheduler)<br>
- <b>[Capacity Scheduler] support simple user and group mappings to queues</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2409">YARN-2409</a>.
- Critical bug reported by Nishan Shetty and fixed by Rohith (resourcemanager)<br>
- <b>Active to StandBy transition does not stop rmDispatcher that causes 1 AsyncDispatcher thread leak. </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2406">YARN-2406</a>.
- Major sub-task reported by Jian He and fixed by Tsuyoshi OZAWA <br>
- <b>Move RM recovery related proto to yarn_server_resourcemanager_recovery.proto</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2405">YARN-2405</a>.
- Major bug reported by Maysam Yabandeh and fixed by Tsuyoshi OZAWA <br>
- <b>NPE in FairSchedulerAppsBlock</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2400">YARN-2400</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>TestAMRestart fails intermittently</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2399">YARN-2399</a>.
- Major improvement reported by Karthik Kambatla and fixed by Karthik Kambatla (fairscheduler)<br>
- <b>FairScheduler: Merge AppSchedulable and FSSchedulerApp into FSAppAttempt</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2397">YARN-2397</a>.
- Critical bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>RM and TS web interfaces sometimes return request is a replay error in secure mode</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2395">YARN-2395</a>.
- Major new feature reported by Ashwin Shankar and fixed by Wei Yan (fairscheduler)<br>
- <b>FairScheduler: Preemption timeout should be configurable per queue</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2394">YARN-2394</a>.
- Major new feature reported by Ashwin Shankar and fixed by Wei Yan (fairscheduler)<br>
- <b>FairScheduler: Configure fairSharePreemptionThreshold per queue</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2393">YARN-2393</a>.
- Major new feature reported by Ashwin Shankar and fixed by Wei Yan (fairscheduler)<br>
- <b>FairScheduler: Add the notion of steady fair share</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2389">YARN-2389</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler , fairscheduler)<br>
- <b>Adding support for drainig a queue, ie killing all apps in the queue</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2388">YARN-2388</a>.
- Major test reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>TestTimelineWebServices fails on trunk after HADOOP-10791</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2387">YARN-2387</a>.
- Blocker bug reported by Mit Desai and fixed by Mit Desai <br>
- <b>Resource Manager crashes with NPE due to lack of synchronization</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2378">YARN-2378</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler)<br>
- <b>Adding support for moving apps between queues in Capacity Scheduler</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2377">YARN-2377</a>.
- Major improvement reported by Gera Shegalov and fixed by Gera Shegalov (nodemanager)<br>
- <b>Localization exception stack traces are not passed as diagnostic info</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2374">YARN-2374</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>YARN trunk build failing TestDistributedShell.testDSShell</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2373">YARN-2373</a>.
- Major bug reported by Larry McCay and fixed by Larry McCay <br>
- <b>WebAppUtils Should Use configuration.getPassword for Accessing SSL Passwords</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2372">YARN-2372</a>.
- Minor improvement reported by Fengdong Yu and fixed by Fengdong Yu (documentation)<br>
- <b>There are Chinese Characters in the FairScheduler's document</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2370">YARN-2370</a>.
- Trivial bug reported by Wenwu Peng and fixed by Wenwu Peng (resourcemanager)<br>
- <b>Fix comment in o.a.h.y.server.resourcemanager.schedulerAppSchedulingInfo</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2363">YARN-2363</a>.
- Major bug reported by Jason Lowe and fixed by Jason Lowe (resourcemanager)<br>
- <b>Submitted applications occasionally lack a tracking URL</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2361">YARN-2361</a>.
- Trivial improvement reported by zhihai xu and fixed by zhihai xu (resourcemanager)<br>
- <b>RMAppAttempt state machine entries for KILLED state has duplicate event entries</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2360">YARN-2360</a>.
- Major new feature reported by Ashwin Shankar and fixed by Ashwin Shankar (fairscheduler)<br>
- <b>Fair Scheduler: Display dynamic fair share for queues on the scheduler page</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2359">YARN-2359</a>.
- Critical bug reported by zhihai xu and fixed by zhihai xu (resourcemanager)<br>
- <b>Application hangs when it fails to launch AM container </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2354">YARN-2354</a>.
- Major sub-task reported by Jian He and fixed by Li Lu <br>
- <b>DistributedShell may allocate more containers than client specified after it restarts</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2352">YARN-2352</a>.
- Major improvement reported by Karthik Kambatla and fixed by Karthik Kambatla (scheduler)<br>
- <b>FairScheduler: Collect metrics on duration of critical methods that affect performance</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2347">YARN-2347</a>.
- Major sub-task reported by Junping Du and fixed by Junping Du <br>
- <b>Consolidate RMStateVersion and NMDBSchemaVersion into StateVersion in yarn-server-common</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2343">YARN-2343</a>.
- Trivial improvement reported by Li Lu and fixed by Li Lu <br>
- <b>Improve error message on token expire exception</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2337">YARN-2337</a>.
- Trivial improvement reported by zhihai xu and fixed by zhihai xu (resourcemanager)<br>
- <b>ResourceManager sets ClientRMService in RMContext multiple times</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2323">YARN-2323</a>.
- Minor improvement reported by Hong Zhiguo and fixed by Hong Zhiguo (fairscheduler)<br>
- <b>FairShareComparator creates too many Resource objects</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2321">YARN-2321</a>.
- Major bug reported by Leitao Guo and fixed by Leitao Guo (nodemanager)<br>
- <b>NodeManager web UI can incorrectly report Pmem enforcement</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2317">YARN-2317</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu (documentation)<br>
- <b>Update documentation about how to write YARN applications</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2314">YARN-2314</a>.
- Critical bug reported by Jason Lowe and fixed by Jason Lowe (client)<br>
- <b>ContainerManagementProtocolProxy can create thousands of threads for a large cluster</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2313">YARN-2313</a>.
- Major bug reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (fairscheduler)<br>
- <b>Livelock can occur in FairScheduler when there are lots of running apps</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2312">YARN-2312</a>.
- Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>Marking ContainerId#getId as deprecated</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2308">YARN-2308</a>.
- Critical bug reported by Wangda Tan and fixed by chang li (resourcemanager , scheduler)<br>
- <b>NPE happened when RM restart after CapacityScheduler queue configuration changed </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2302">YARN-2302</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Refactor TimelineWebServices</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2298">YARN-2298</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (client)<br>
- <b>Move TimelineClient to yarn-common project</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2295">YARN-2295</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu <br>
- <b>Refactor YARN distributed shell with existing public stable API</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2288">YARN-2288</a>.
- Major sub-task reported by Junping Du and fixed by Junping Du (timelineserver)<br>
- <b>Data persistent in timelinestore should be versioned</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2279">YARN-2279</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Add UTs to cover timeline server authentication</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2277">YARN-2277</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
- <b>Add Cross-Origin support to the ATS REST API</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2274">YARN-2274</a>.
- Trivial improvement reported by Karthik Kambatla and fixed by Karthik Kambatla (fairscheduler)<br>
- <b>FairScheduler: Add debug information about cluster capacity, availability and reservations</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2273">YARN-2273</a>.
- Major bug reported by Andy Skelton and fixed by Wei Yan (fairscheduler , resourcemanager)<br>
- <b>NPE in ContinuousScheduling thread when we lose a node</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2269">YARN-2269</a>.
- Major bug reported by Yesha Vora and fixed by Craig Welch <br>
- <b>External links need to be removed from YARN UI</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2264">YARN-2264</a>.
- Major bug reported by Siddharth Seth and fixed by Li Lu <br>
- <b>Race in DrainDispatcher can cause random test failures</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2260">YARN-2260</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>Add containers to launchedContainers list in RMNode on container recovery</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2252">YARN-2252</a>.
- Major bug reported by Ratandeep Ratti and fixed by (scheduler)<br>
- <b>Intermittent failure of TestFairScheduler.testContinuousScheduling</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2251">YARN-2251</a>.
- Major bug reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>Avoid negative elapsed time in JHS/MRAM web UI and services</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2249">YARN-2249</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>AM release request may be lost on RM restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2248">YARN-2248</a>.
- Major sub-task reported by Janos Matyas and fixed by Janos Matyas (capacityscheduler)<br>
- <b>Capacity Scheduler changes for moving apps between queues</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2244">YARN-2244</a>.
- Critical bug reported by Anubhav Dhoot and fixed by Anubhav Dhoot (fairscheduler)<br>
- <b>FairScheduler missing handling of containers for unknown application attempts </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2242">YARN-2242</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu <br>
- <b>Improve exception information on AM launch crashes</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2237">YARN-2237</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>MRAppMaster changes for AMRMToken roll-up</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2229">YARN-2229</a>.
- Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>ContainerId can overflow with RM restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2228">YARN-2228</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>TimelineServer should load pseudo authentication filter when authentication = simple</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2219">YARN-2219</a>.
- Major bug reported by Ashwin Shankar and fixed by Jian He (resourcemanager)<br>
- <b>AMs and NMs can get exceptions after recovery but before scheduler knowns apps and app-attempts</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2214">YARN-2214</a>.
- Major improvement reported by Ashwin Shankar and fixed by Ashwin Shankar (scheduler)<br>
- <b>FairScheduler: preemptContainerPreCheck() in FSParentQueue delays convergence towards fairness</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2212">YARN-2212</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>ApplicationMaster needs to find a way to update the AMRMToken periodically</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2211">YARN-2211</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>RMStateStore needs to save AMRMToken master key for recovery when RM restart/failover happens </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2209">YARN-2209</a>.
- Major improvement reported by Jian He and fixed by Jian He <br>
- <b>Replace AM resync/shutdown command with corresponding exceptions</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2208">YARN-2208</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>AMRMTokenManager need to have a way to roll over AMRMToken</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2207">YARN-2207</a>.
- Major task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>Add ability to roll over AMRMToken</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2198">YARN-2198</a>.
- Major improvement reported by Remus Rusanu and fixed by Remus Rusanu <br>
- <b>Remove the need to run NodeManager as privileged account for Windows Secure Container Executor</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2197">YARN-2197</a>.
- Minor improvement reported by Akira AJISAKA and fixed by Akira AJISAKA (documentation)<br>
- <b>Add a link to YARN CHANGES.txt in the left side of doc</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2182">YARN-2182</a>.
- Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
- <b>Update ContainerId#toString() to avoid conflicts before and after RM restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2181">YARN-2181</a>.
- Major bug reported by Wangda Tan and fixed by Wangda Tan (resourcemanager , webapp)<br>
- <b>Add preemption info to RM Web UI and add logs when preemption occurs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2174">YARN-2174</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>Enabling HTTPs for the writer REST API of TimelineServer</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2161">YARN-2161</a>.
- Major bug reported by Binglin Chang and fixed by Binglin Chang <br>
- <b>Fix build on macosx: YARN parts</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2158">YARN-2158</a>.
- Minor test reported by Ted Yu and fixed by Varun Vasudev <br>
- <b>TestRMWebServicesAppsModification sometimes fails in trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2153">YARN-2153</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>Ensure distributed shell work with RM work-preserving recovery</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2147">YARN-2147</a>.
- Minor bug reported by Jason Lowe and fixed by Chen He (resourcemanager)<br>
- <b>client lacks delegation token exception details when application submit fails</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2138">YARN-2138</a>.
- Major bug reported by Jian He and fixed by Varun Saxena <br>
- <b>Cleanup notifyDone* methods in RMStateStore</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2131">YARN-2131</a>.
- Major new feature reported by Karthik Kambatla and fixed by Robert Kanter (resourcemanager)<br>
- <b>Add a way to format the RMStateStore</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2102">YARN-2102</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>More generalized timeline ACLs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2088">YARN-2088</a>.
- Major bug reported by Binglin Chang and fixed by Binglin Chang <br>
- <b>Fix code bug in GetApplicationsRequestPBImpl#mergeLocalToBuilder</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2080">YARN-2080</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (resourcemanager)<br>
- <b>Admission Control: Integrate Reservation subsystem with ResourceManager</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2070">YARN-2070</a>.
- Minor sub-task reported by Zhijie Shen and fixed by Robert Kanter <br>
- <b>DistributedShell publishes unfriendly user information to the timeline server</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2045">YARN-2045</a>.
- Major sub-task reported by Junping Du and fixed by Junping Du (nodemanager)<br>
- <b>Data persisted in NM should be versioned</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2035">YARN-2035</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles <br>
- <b>FileSystemApplicationHistoryStore blocks RM and AHS while NN is in safemode</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2034">YARN-2034</a>.
- Minor bug reported by Jason Lowe and fixed by Chen He (nodemanager)<br>
- <b>Description for yarn.nodemanager.localizer.cache.target-size-mb is incorrect</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2033">YARN-2033</a>.
- Major sub-task reported by Vinod Kumar Vavilapalli and fixed by Zhijie Shen <br>
- <b>Merging generic-history into the Timeline Store</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2026">YARN-2026</a>.
- Major bug reported by Ashwin Shankar and fixed by Ashwin Shankar (scheduler)<br>
- <b>Fair scheduler: Consider only active queues for computing fairshare</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2013">YARN-2013</a>.
- Major sub-task reported by Zhijie Shen and fixed by Tsuyoshi OZAWA (nodemanager)<br>
- <b>The diagnostics is always the ExitCodeException stack when the container crashes</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2010">YARN-2010</a>.
- Blocker bug reported by bc Wong and fixed by Karthik Kambatla (resourcemanager)<br>
- <b>Handle app-recovery failures gracefully</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2008">YARN-2008</a>.
- Major sub-task reported by Chen He and fixed by Craig Welch <br>
- <b>CapacityScheduler may report incorrect queueMaxCap if there is hierarchy queue structure </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2001">YARN-2001</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>Threshold for RM to accept requests from AM after failover</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1994">YARN-1994</a>.
- Major improvement reported by Arpit Agarwal and fixed by Craig Welch (nodemanager , resourcemanager , webapp)<br>
- <b>Expose YARN/MR endpoints on multiple interfaces</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1972">YARN-1972</a>.
- Major sub-task reported by Remus Rusanu and fixed by Remus Rusanu (nodemanager)<br>
- <b>Implement secure Windows Container Executor</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1964">YARN-1964</a>.
- Major new feature reported by Arun C Murthy and fixed by Abin Shahab <br>
- <b>Create Docker analog of the LinuxContainerExecutor in YARN</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1959">YARN-1959</a>.
- Major bug reported by Sandy Ryza and fixed by Anubhav Dhoot <br>
- <b>Fix headroom calculation in FairScheduler</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1954">YARN-1954</a>.
- Major improvement reported by Zhijie Shen and fixed by Tsuyoshi OZAWA (client)<br>
- <b>Add waitFor to AMRMClient(Async)</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1922">YARN-1922</a>.
- Major bug reported by Billie Rinaldi and fixed by Billie Rinaldi (nodemanager)<br>
- <b>Process group remains alive after container process is killed externally</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1919">YARN-1919</a>.
- Minor bug reported by Devaraj K and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>Potential NPE in EmbeddedElectorService#stop</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1918">YARN-1918</a>.
- Trivial improvement reported by Devaraj K and fixed by Anandha L Ranganathan <br>
- <b>Typo in description and error message for 'yarn.resourcemanager.cluster-id'</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1915">YARN-1915</a>.
- Blocker sub-task reported by Hitesh Shah and fixed by Jason Lowe <br>
- <b>ClientToAMTokenMasterKey should be provided to AM at launch time</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1879">YARN-1879</a>.
- Critical sub-task reported by Jian He and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>Mark Idempotent/AtMostOnce annotations to ApplicationMasterProtocol for RM fail over</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1857">YARN-1857</a>.
- Critical sub-task reported by Thomas Graves and fixed by Chen He (capacityscheduler)<br>
- <b>CapacityScheduler headroom doesn't account for other AM's running</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1796">YARN-1796</a>.
- Minor bug reported by Aaron T. Myers and fixed by Aaron T. Myers (nodemanager)<br>
- <b>container-executor shouldn't require o-r permissions</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1779">YARN-1779</a>.
- Blocker sub-task reported by Karthik Kambatla and fixed by Jian He (resourcemanager)<br>
- <b>Handle AMRMTokens across RM failover</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1769">YARN-1769</a>.
- Major improvement reported by Thomas Graves and fixed by Thomas Graves (capacityscheduler)<br>
- <b>CapacityScheduler: Improve reservations</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1712">YARN-1712</a>.
- Major sub-task reported by Carlo Curino and fixed by Carlo Curino (capacityscheduler , resourcemanager)<br>
- <b>Admission Control: plan follower</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1711">YARN-1711</a>.
- Major sub-task reported by Carlo Curino and fixed by Carlo Curino <br>
- <b>CapacityOverTimePolicy: a policy to enforce quotas over time for YARN-1709</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1710">YARN-1710</a>.
- Major sub-task reported by Carlo Curino and fixed by Carlo Curino (resourcemanager)<br>
- <b>Admission Control: agents to allocate reservation</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1709">YARN-1709</a>.
- Major sub-task reported by Carlo Curino and fixed by Subru Krishnan (resourcemanager)<br>
- <b>Admission Control: Reservation subsystem</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1708">YARN-1708</a>.
- Major sub-task reported by Carlo Curino and fixed by Subru Krishnan (resourcemanager)<br>
- <b>Add a public API to reserve resources (part of YARN-1051)</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1707">YARN-1707</a>.
- Major sub-task reported by Carlo Curino and fixed by Carlo Curino (capacityscheduler)<br>
- <b>Making the CapacityScheduler more dynamic</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1372">YARN-1372</a>.
- Major sub-task reported by Bikas Saha and fixed by Anubhav Dhoot (resourcemanager)<br>
- <b>Ensure all completed containers are reported to the AMs across RM restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1370">YARN-1370</a>.
- Major sub-task reported by Bikas Saha and fixed by Anubhav Dhoot (resourcemanager)<br>
- <b>Fair scheduler to re-populate container allocation state</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1367">YARN-1367</a>.
- Major sub-task reported by Bikas Saha and fixed by Anubhav Dhoot (resourcemanager)<br>
- <b>After restart NM should resync with the RM without killing containers</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1354">YARN-1354</a>.
- Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
- <b>Recover applications upon nodemanager restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1342">YARN-1342</a>.
- Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
- <b>Recover container tokens upon nodemanager restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1341">YARN-1341</a>.
- Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
- <b>Recover NMTokens upon nodemanager restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1337">YARN-1337</a>.
- Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
- <b>Recover containers upon nodemanager restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1326">YARN-1326</a>.
- Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
- <b>RM should log using RMStore at startup time</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1250">YARN-1250</a>.
- Major sub-task reported by Vinod Kumar Vavilapalli and fixed by Zhijie Shen <br>
- <b>Generic history service should support application-acls</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1063">YARN-1063</a>.
- Major sub-task reported by Kyle Leckie and fixed by Remus Rusanu (nodemanager)<br>
- <b>Winutils needs ability to create task as domain user</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1051">YARN-1051</a>.
- Major improvement reported by Carlo Curino and fixed by Carlo Curino (capacityscheduler , resourcemanager , scheduler)<br>
- <b>YARN Admission Control/Planner: enhancing the resource allocation model with time.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-668">YARN-668</a>.
- Blocker sub-task reported by Siddharth Seth and fixed by Junping Du <br>
- <b>TokenIdentifier serialization should consider Unknown fields</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-611">YARN-611</a>.
- Major sub-task reported by Chris Riccomini and fixed by Xuan Gong (resourcemanager)<br>
- <b>Add an AM retry count reset window to YARN RM</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-415">YARN-415</a>.
- Major new feature reported by Kendall Thrapp and fixed by Eric Payne (resourcemanager)<br>
- <b>Capture aggregate memory allocation at the app-level for chargeback</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-90">YARN-90</a>.
- Major sub-task reported by Ravi Gummadi and fixed by Varun Vasudev (nodemanager)<br>
- <b>NodeManager should identify failed disks becoming good again</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6156">MAPREDUCE-6156</a>.
- Blocker bug reported by Sidharta Seethana and fixed by Junping Du <br>
- <b>Fetcher - connect() doesn't handle connection refused correctly </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6142">MAPREDUCE-6142</a>.
- Critical sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>Test failure in TestJobHistoryEventHandler and TestMRTimelineEventHandling</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6126">MAPREDUCE-6126</a>.
- Major bug reported by Junping Du and fixed by Junping Du <br>
- <b>(Rumen) Rumen tool returns error "ava.lang.IllegalArgumentException: JobBuilder.process(HistoryEvent): unknown event type"</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6125">MAPREDUCE-6125</a>.
- Major bug reported by Mit Desai and fixed by Mit Desai (test)<br>
- <b>TestContainerLauncherImpl sometimes fails</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6123">MAPREDUCE-6123</a>.
- Trivial bug reported by Chris Nauroth and fixed by Chris Nauroth (test)<br>
- <b>TestCombineFileInputFormat incorrectly starts 2 MiniDFSCluster instances.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6122">MAPREDUCE-6122</a>.
- Trivial bug reported by Chris Nauroth and fixed by Chris Nauroth (test)<br>
- <b>TestLineRecordReader may fail due to test data files checked out of git with incorrect line endings.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6115">MAPREDUCE-6115</a>.
- Minor test reported by Ted Yu and fixed by Binglin Chang <br>
- <b>TestPipeApplication#testSubmitter fails in trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6109">MAPREDUCE-6109</a>.
- Trivial bug reported by Charles Lamb and fixed by Charles Lamb (distcp)<br>
- <b>Fix minor typo in distcp -p usage text</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6104">MAPREDUCE-6104</a>.
- Major bug reported by Mit Desai and fixed by Mit Desai <br>
- <b>TestJobHistoryParsing.testPartialJob fails in branch-2</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6095">MAPREDUCE-6095</a>.
- Major bug reported by Gera Shegalov and fixed by Gera Shegalov (applicationmaster , distributed-cache)<br>
- <b>Enable DistributedCache for uber-mode Jobs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6094">MAPREDUCE-6094</a>.
- Minor bug reported by Sangjin Lee and fixed by Akira AJISAKA (test)<br>
- <b>TestMRCJCFileInputFormat.testAddInputPath() fails on trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6093">MAPREDUCE-6093</a>.
- Trivial bug reported by Charles Lamb and fixed by Charles Lamb (distcp , documentation)<br>
- <b>minor distcp doc edits</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6091">MAPREDUCE-6091</a>.
- Major bug reported by Sangjin Lee and fixed by Sangjin Lee (client)<br>
- <b>YARNRunner.getJobStatus() fails with ApplicationNotFoundException if the job rolled off the RM view</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6090">MAPREDUCE-6090</a>.
- Major bug reported by Robert Kanter and fixed by Robert Kanter (client)<br>
- <b>mapred hsadmin getGroups fails to connect in some cases</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6086">MAPREDUCE-6086</a>.
- Major improvement reported by zhihai xu and fixed by zhihai xu (security)<br>
- <b>mapreduce.job.credentials.binary should allow all URIs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6075">MAPREDUCE-6075</a>.
- Major bug reported by Jason Lowe and fixed by Jason Lowe (jobhistoryserver)<br>
- <b>HistoryServerFileSystemStateStore can create zero-length files</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6073">MAPREDUCE-6073</a>.
- Trivial bug reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (documentation)<br>
- <b>Description of mapreduce.job.speculative.slowtaskthreshold in mapred-default should be moved into description tags</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6072">MAPREDUCE-6072</a>.
- Minor improvement reported by Akira AJISAKA and fixed by Akira AJISAKA (documentation)<br>
- <b>Remove INSTALL document</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6071">MAPREDUCE-6071</a>.
- Trivial improvement reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (client)<br>
- <b>JobImpl#makeUberDecision doesn't log that Uber mode is disabled because of too much CPUs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6070">MAPREDUCE-6070</a>.
- Trivial improvement reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (documentation)<br>
- <b>yarn.app.am.resource.mb/cpu-vcores affects uber mode but is not documented</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6063">MAPREDUCE-6063</a>.
- Major bug reported by zhihai xu and fixed by zhihai xu (mrv1 , mrv2)<br>
- <b>In sortAndSpill of MapTask.java, size is calculated wrongly when bufend < bufstart.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6052">MAPREDUCE-6052</a>.
- Major bug reported by Junping Du and fixed by Junping Du <br>
- <b>Support overriding log4j.properties per job</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6051">MAPREDUCE-6051</a>.
- Trivial bug reported by Ray Chiang and fixed by Ray Chiang <br>
- <b>Fix typos in log messages</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6048">MAPREDUCE-6048</a>.
- Minor test reported by Ted Yu and fixed by Varun Vasudev <br>
- <b>TestJavaSerialization fails in trunk build</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6044">MAPREDUCE-6044</a>.
- Major bug reported by Zhijie Shen and fixed by Zhijie Shen (jobhistoryserver)<br>
- <b>Fully qualified intermediate done directory will break per-user dir creation on Windows</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6036">MAPREDUCE-6036</a>.
- Major bug reported by Mit Desai and fixed by chang li
<TRUNCATED>
[49/50] [abbrv] hadoop git commit: YARN-1462. Correct fix version
from branch-2.7.1 to branch-2.8 in CHANGES.txt
Posted by zj...@apache.org.
YARN-1462. Correct fix version from branch-2.7.1 to branch-2.8 in
CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7505e35e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7505e35e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7505e35e
Branch: refs/heads/YARN-2928
Commit: 7505e35e56a0f6e91d50f00d9c915aa46dca4fff
Parents: 850e488
Author: Xuan <xg...@apache.org>
Authored: Mon Jun 1 11:39:00 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:22 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7505e35e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 807165f..543d9cf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -374,6 +374,9 @@ Release 2.8.0 - UNRELEASED
YARN-3467. Expose allocatedMB, allocatedVCores, and runningContainers metrics on
running Applications in RM Web UI. (Anubhav Dhoot via kasha)
+ YARN-1462. Made RM write application tags to timeline server and exposed them
+ to users via generic history web UI and REST API. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
@@ -584,9 +587,6 @@ Release 2.7.1 - UNRELEASED
YARN-3723. Need to clearly document primaryFilter and otherInfo value type.
(Zhijie Shen via xgong)
- YARN-1462. Made RM write application tags to timeline server and exposed them
- to users via generic history web UI and REST API. (Xuan Gong via zjshen)
-
OPTIMIZATIONS
BUG FIXES
[18/50] [abbrv] hadoop git commit: HDFS-8429. Avoid stuck threads if
there is an error in DomainSocketWatcher that stops the thread. (zhouyingchao
via cmccabe)
Posted by zj...@apache.org.
HDFS-8429. Avoid stuck threads if there is an error in DomainSocketWatcher that stops the thread. (zhouyingchao via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d5c0d4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d5c0d4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d5c0d4a
Branch: refs/heads/YARN-2928
Commit: 3d5c0d4a2a8d733bcd6a95a85932a2a6898c20b4
Parents: 61510b9
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu May 28 11:52:28 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/net/unix/DomainSocketWatcher.java | 21 +++++-
.../hadoop/net/unix/DomainSocketWatcher.c | 2 +-
.../net/unix/TestDomainSocketWatcher.java | 75 ++++++++++++++++++++
4 files changed, 99 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5c0d4a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 69f5aa7..51eff78 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -782,6 +782,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12035. shellcheck plugin displays a wrong version potentially
(Kengo Seki via aw)
+ HDFS-8429. Avoid stuck threads if there is an error in DomainSocketWatcher
+ that stops the thread. (zhouyingchao via cmccabe)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5c0d4a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
index 5648ae1..ad2fbfb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
@@ -470,6 +470,7 @@ public final class DomainSocketWatcher implements Closeable {
// Handle pending additions (before pending removes).
for (Iterator<Entry> iter = toAdd.iterator(); iter.hasNext(); ) {
Entry entry = iter.next();
+ iter.remove();
DomainSocket sock = entry.getDomainSocket();
Entry prevEntry = entries.put(sock.fd, entry);
Preconditions.checkState(prevEntry == null,
@@ -479,7 +480,6 @@ public final class DomainSocketWatcher implements Closeable {
LOG.trace(this + ": adding fd " + sock.fd);
}
fdSet.add(sock.fd);
- iter.remove();
}
// Handle pending removals
while (true) {
@@ -525,6 +525,25 @@ public final class DomainSocketWatcher implements Closeable {
}
entries.clear();
fdSet.close();
+ closed = true;
+ if (!(toAdd.isEmpty() && toRemove.isEmpty())) {
+ // Items in toAdd might not be added to entries, handle it here
+ for (Iterator<Entry> iter = toAdd.iterator(); iter.hasNext();) {
+ Entry entry = iter.next();
+ entry.getDomainSocket().refCount.unreference();
+ entry.getHandler().handle(entry.getDomainSocket());
+ IOUtils.cleanup(LOG, entry.getDomainSocket());
+ iter.remove();
+ }
+ // Items in toRemove might not be really removed, handle it here
+ while (true) {
+ Map.Entry<Integer, DomainSocket> entry = toRemove.firstEntry();
+ if (entry == null)
+ break;
+ sendCallback("close", entries, fdSet, entry.getValue().fd);
+ }
+ }
+ processedCond.signalAll();
} finally {
lock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5c0d4a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c
index 596601b..82e6af5 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocketWatcher.c
@@ -212,7 +212,7 @@ done:
free(carr);
if (jthr) {
(*env)->DeleteLocalRef(env, jarr);
- jarr = NULL;
+ (*env)->Throw(env, jthr);
}
return jarr;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d5c0d4a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
index 4b0e2a8..4cc86a7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocketWatcher.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse;
import java.util.ArrayList;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
@@ -181,6 +182,80 @@ public class TestDomainSocketWatcher {
watcher.close();
}
+ @Test(timeout = 300000)
+ public void testStressInterruption() throws Exception {
+ final int SOCKET_NUM = 250;
+ final ReentrantLock lock = new ReentrantLock();
+ final DomainSocketWatcher watcher = newDomainSocketWatcher(10);
+ final ArrayList<DomainSocket[]> pairs = new ArrayList<DomainSocket[]>();
+ final AtomicInteger handled = new AtomicInteger(0);
+
+ final Thread adderThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ for (int i = 0; i < SOCKET_NUM; i++) {
+ DomainSocket pair[] = DomainSocket.socketpair();
+ watcher.add(pair[1], new DomainSocketWatcher.Handler() {
+ @Override
+ public boolean handle(DomainSocket sock) {
+ handled.incrementAndGet();
+ return true;
+ }
+ });
+ lock.lock();
+ try {
+ pairs.add(pair);
+ } finally {
+ lock.unlock();
+ }
+ TimeUnit.MILLISECONDS.sleep(1);
+ }
+ } catch (Throwable e) {
+ LOG.error(e);
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+ final Thread removerThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ final Random random = new Random();
+ try {
+ while (handled.get() != SOCKET_NUM) {
+ lock.lock();
+ try {
+ if (!pairs.isEmpty()) {
+ int idx = random.nextInt(pairs.size());
+ DomainSocket pair[] = pairs.remove(idx);
+ if (random.nextBoolean()) {
+ pair[0].close();
+ } else {
+ watcher.remove(pair[1]);
+ }
+ TimeUnit.MILLISECONDS.sleep(1);
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+ } catch (Throwable e) {
+ LOG.error(e);
+ throw new RuntimeException(e);
+ }
+ }
+ });
+
+ adderThread.start();
+ removerThread.start();
+ TimeUnit.MILLISECONDS.sleep(100);
+ watcher.watcherThread.interrupt();
+ Uninterruptibles.joinUninterruptibly(adderThread);
+ Uninterruptibles.joinUninterruptibly(removerThread);
+ Uninterruptibles.joinUninterruptibly(watcher.watcherThread);
+ }
+
/**
* Creates a new DomainSocketWatcher and tracks its thread for termination due
* to an unexpected exception. At the end of each test, if there was an
[44/50] [abbrv] hadoop git commit: Revert "YARN-2900. Application
(Attempt and Container) Not Found in AHS results"
Posted by zj...@apache.org.
Revert "YARN-2900. Application (Attempt and Container) Not Found in AHS results"
This reverts commit 06f8e9cabaf3c05cd7d16215cff47265ea773f39.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00ccf736
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00ccf736
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00ccf736
Branch: refs/heads/YARN-2928
Commit: 00ccf736966b2980fd34756ad30f5f84d48efd9d
Parents: c0ecae3
Author: Xuan <xg...@apache.org>
Authored: Sun May 31 00:26:04 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:13:00 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 -
.../timeline/webapp/TimelineWebServices.java | 2 +
.../TestApplicationHistoryClientService.java | 70 +-------------------
.../hadoop/yarn/server/webapp/WebServices.java | 27 +++-----
4 files changed, 13 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00ccf736/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 45c4f68..6f5faa9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -694,9 +694,6 @@ Release 2.7.1 - UNRELEASED
YARN-3686. CapacityScheduler should trim default_node_label_expression.
(Sunil G via wangda)
- YARN-2900. Application (Attempt and Container) Not Found in AHS results
- in InternalServer Error (500). (Zhijie Shen and Mit Desai via xgong)
-
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00ccf736/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
index 40e9fed..90f4d39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
@@ -47,6 +47,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@@ -60,6 +61,7 @@ import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00ccf736/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index d04903c..764ecb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -41,9 +41,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
@@ -57,13 +54,12 @@ public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService;
private static TimelineDataManager dataManager;
- private final static int MAX_APPS = 2;
@BeforeClass
public static void setup() throws Exception {
Configuration conf = new YarnConfiguration();
TimelineStore store =
- TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
+ TestApplicationHistoryManagerOnTimelineStore.createStore(2);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
dataManager =
new TimelineDataManager(store, aclsManager);
@@ -76,70 +72,6 @@ public class TestApplicationHistoryClientService {
}
@Test
- public void testApplicationNotFound() throws IOException, YarnException {
- ApplicationId appId = null;
- appId = ApplicationId.newInstance(0, MAX_APPS + 1);
- GetApplicationReportRequest request =
- GetApplicationReportRequest.newInstance(appId);
- try {
- @SuppressWarnings("unused")
- GetApplicationReportResponse response =
- clientService.getApplicationReport(request);
- Assert.fail("Exception should have been thrown before we reach here.");
- } catch (ApplicationNotFoundException e) {
- //This exception is expected.
- Assert.assertTrue(e.getMessage().contains(
- "doesn't exist in the timeline store"));
- } catch (Exception e) {
- Assert.fail("Undesired exception caught");
- }
- }
-
- @Test
- public void testApplicationAttemptNotFound() throws IOException, YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 1);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
- GetApplicationAttemptReportRequest request =
- GetApplicationAttemptReportRequest.newInstance(appAttemptId);
- try {
- @SuppressWarnings("unused")
- GetApplicationAttemptReportResponse response =
- clientService.getApplicationAttemptReport(request);
- Assert.fail("Exception should have been thrown before we reach here.");
- } catch (ApplicationAttemptNotFoundException e) {
- //This Exception is expected
- System.out.println(e.getMessage());
- Assert.assertTrue(e.getMessage().contains(
- "doesn't exist in the timeline store"));
- } catch (Exception e) {
- Assert.fail("Undesired exception caught");
- }
- }
-
- @Test
- public void testContainerNotFound() throws IOException, YarnException {
- ApplicationId appId = ApplicationId.newInstance(0, 1);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 1);
- ContainerId containerId = ContainerId.newContainerId(appAttemptId,
- MAX_APPS + 1);
- GetContainerReportRequest request =
- GetContainerReportRequest.newInstance(containerId);
- try {
- @SuppressWarnings("unused")
- GetContainerReportResponse response =
- clientService.getContainerReport(request);
- } catch (ContainerNotFoundException e) {
- //This exception is expected
- Assert.assertTrue(e.getMessage().contains(
- "doesn't exist in the timeline store"));
- } catch (Exception e) {
- Assert.fail("Undesired exception caught");
- }
- }
-
- @Test
public void testApplicationReport() throws IOException, YarnException {
ApplicationId appId = null;
appId = ApplicationId.newInstance(0, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00ccf736/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index ae27d45..8404719 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -47,9 +47,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
-import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@@ -480,21 +477,17 @@ public class WebServices {
private static void rewrapAndThrowException(Exception e) {
if (e instanceof UndeclaredThrowableException) {
- rewrapAndThrowThrowable(e.getCause());
- } else {
- rewrapAndThrowThrowable(e);
- }
- }
-
- private static void rewrapAndThrowThrowable(Throwable t) {
- if (t instanceof AuthorizationException) {
- throw new ForbiddenException(t);
- } if (t instanceof ApplicationNotFoundException ||
- t instanceof ApplicationAttemptNotFoundException ||
- t instanceof ContainerNotFoundException) {
- throw new NotFoundException(t);
+ if (e.getCause() instanceof AuthorizationException) {
+ throw new ForbiddenException(e.getCause());
+ } else {
+ throw new WebApplicationException(e.getCause());
+ }
} else {
- throw new WebApplicationException(t);
+ if (e instanceof AuthorizationException) {
+ throw new ForbiddenException(e);
+ } else {
+ throw new WebApplicationException(e);
+ }
}
}
[50/50] [abbrv] hadoop git commit: HDFS-8256. '-storagepolicies ,
-blockId ,
-replicaDetails ' options are missed out in usage and from documentation
(Contributed by J.Andreina)
Posted by zj...@apache.org.
HDFS-8256. '-storagepolicies , -blockId ,-replicaDetails ' options are missed out in usage and from documentation (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e124808
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e124808
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e124808
Branch: refs/heads/YARN-2928
Commit: 2e1248088731e53bc085471c4799a23097e82a05
Parents: 77c1692
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 2 16:07:18 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:23 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java | 9 +++++----
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 6 +++++-
3 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e124808/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1f7a36a..fa28a16 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -831,6 +831,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8490. Typo in trace enabled log in ExceptionHandler of WebHDFS.
(Archana T via ozawa)
+ HDFS-8256. "-storagepolicies , -blockId ,-replicaDetails " options are missed
+ out in usage and from documentation (J.Andreina via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e124808/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 67386ac..33033e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -78,8 +78,9 @@ public class DFSck extends Configured implements Tool {
private static final String USAGE = "Usage: hdfs fsck <path> "
+ "[-list-corruptfileblocks | "
+ "[-move | -delete | -openforwrite] "
- + "[-files [-blocks [-locations | -racks]]]] "
- + "[-includeSnapshots] [-showprogress]\n"
+ + "[-files [-blocks [-locations | -racks | -replicaDetails]]]] "
+ + "[-includeSnapshots] [-showprogress] "
+ + "[-storagepolicies] [-blockId <blk_Id>]\n"
+ "\t<path>\tstart checking from this path\n"
+ "\t-move\tmove corrupted files to /lost+found\n"
+ "\t-delete\tdelete corrupted files\n"
@@ -94,12 +95,12 @@ public class DFSck extends Configured implements Tool {
+ "\t-files -blocks -locations\tprint out locations for every block\n"
+ "\t-files -blocks -racks"
+ "\tprint out network topology for data-node locations\n"
+ + "\t-files -blocks -replicaDetails\tprint out each replica details \n"
+ "\t-storagepolicies\tprint out storage policy summary for the blocks\n"
+ "\t-showprogress\tshow progress in output. Default is OFF (no progress)\n"
+ "\t-blockId\tprint out which file this blockId belongs to, locations"
+ " (nodes, racks) of this block, and other diagnostics info"
- + " (under replicated, corrupted or not, etc)\n"
- + "\t-replicaDetails\tprint out each replica details \n\n"
+ + " (under replicated, corrupted or not, etc)\n\n"
+ "Please Note:\n"
+ "\t1. By default fsck ignores files opened for write, "
+ "use -openforwrite to report such files. They are usually "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e124808/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 2ceff89..2121958 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -99,8 +99,9 @@ Usage:
hdfs fsck <path>
[-list-corruptfileblocks |
[-move | -delete | -openforwrite]
- [-files [-blocks [-locations | -racks]]]
+ [-files [-blocks [-locations | -racks | -replicaDetails]]]
[-includeSnapshots] [-showprogress]
+ [-storagepolicies] [-blockId <blk_Id>]
| COMMAND\_OPTION | Description |
|:---- |:---- |
@@ -110,11 +111,14 @@ Usage:
| `-files` `-blocks` | Print out the block report |
| `-files` `-blocks` `-locations` | Print out locations for every block. |
| `-files` `-blocks` `-racks` | Print out network topology for data-node locations. |
+| `-files` `-blocks` `-replicaDetails` | Print out each replica details. |
| `-includeSnapshots` | Include snapshot data if the given path indicates a snapshottable directory or there are snapshottable directories under it. |
| `-list-corruptfileblocks` | Print out list of missing blocks and files they belong to. |
| `-move` | Move corrupted files to /lost+found. |
| `-openforwrite` | Print out files opened for write. |
| `-showprogress` | Print out dots for progress in output. Default is OFF (no progress). |
+| `-storagepolicies` | Print out storage policy summary for the blocks. |
+| `-blockId` | Print out information about the block. |
Runs the HDFS filesystem checking utility. See [fsck](./HdfsUserGuide.html#fsck) for more info.
[12/50] [abbrv] hadoop git commit: YARN-3723. Need to clearly
document primaryFilter and otherInfo value type. Contributed by Zhijie Shen
Posted by zj...@apache.org.
YARN-3723. Need to clearly document primaryFilter and otherInfo value
type. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de39fedb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de39fedb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de39fedb
Branch: refs/heads/YARN-2928
Commit: de39fedb5ca44b62aaef7400747e2a1abc312f41
Parents: ac3000a
Author: Xuan <xg...@apache.org>
Authored: Thu May 28 10:19:36 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../src/site/markdown/TimelineServer.md | 20 ++++++++++++++++----
2 files changed, 19 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de39fedb/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5326f2d..1dd4ee3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -566,6 +566,9 @@ Release 2.7.1 - UNRELEASED
YARN-3539. Updated timeline server documentation and marked REST APIs evolving.
(Steve Loughran via zjshen)
+ YARN-3723. Need to clearly document primaryFilter and otherInfo value type.
+ (Zhijie Shen via xgong)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de39fedb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index 5a18fb2..90f9512 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -648,11 +648,20 @@ Use the following URI to obtain all the entity objects of a given
the set of fields contains `LAST_EVENT_ONLY` and not `EVENTS`, the most recent
event for each entity is retrieved. If null, retrieves all fields.
+Note that the value of the key/value pair for `primaryFilter` and
+`secondaryFilters` parameters can be of different data types, and matching is
+data type sensitive. Users need to format the value properly. For example, `123`
+and `"123"` means an integer and a string respectively. If the entity has a
+string `"123"` for `primaryFilter`, but the parameter is set to the integer
+`123`, the entity will not be matched. Similarly, `true` means a boolean while
+`"true"` means a string. In general, the value will be casted as a certain Java
+type in consistent with `jackson` library parsing a JSON clip.
+
### Elements of the `entities` (Timeline Entity List) Object
When you make a request for the list of timeline entities, the information
will be returned as a collection of container objects. See also
-{{Timeline Entity}} for syntax of the timeline entity object.
+`Timeline Entity` for syntax of the timeline entity object.
| Item | Data Type | Description|
|:---- |:---- |:---- |
@@ -751,7 +760,9 @@ Use the following URI to obtain the entity object identified by the
### Elements of the `entity` (Timeline Entity) Object:
-See also {{Timeline Event List}} for syntax of the timeline event object.
+See also `Timeline Event List` for syntax of the timeline event object. Note
+that `value` of `primaryfilters` and `otherinfo` is an Object instead of a
+String.
| Item | Data Type | Description|
@@ -841,7 +852,8 @@ will be returned as a collection of event objects.
|:---- |:---- |:---- |
| `events` | array of timeline event objects(JSON) | The collection of timeline event objects |
-Below is the elements of a single event object.
+Below is the elements of a single event object. Note that `value` of
+`eventinfo` and `otherinfo` is an Object instead of a String.
| Item | Data Type | Description|
@@ -1547,7 +1559,7 @@ None
### Elements of the `containers` (Container List) Object
When you make a request for the list of containers, the information will be
-returned as a collection of container objects. See also {{Container}} for
+returned as a collection of container objects. See also `Container` for
syntax of the container object.
| Item | Data Type | Description |
[09/50] [abbrv] hadoop git commit: YARN-3700. Made generic history
service load a number of latest applications according to the parameter or
the configuration. Contributed by Xuan Gong.
Posted by zj...@apache.org.
YARN-3700. Made generic history service load a number of latest applications according to the parameter or the configuration. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81dd9e80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81dd9e80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81dd9e80
Branch: refs/heads/YARN-2928
Commit: 81dd9e80b32b89cbee312b7716b20da7dc8c06b5
Parents: 6fdef78
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed May 27 16:51:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:53 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +++++++
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../src/main/resources/yarn-default.xml | 9 +++++++
.../ApplicationHistoryClientService.java | 2 +-
.../ApplicationHistoryManager.java | 12 ++++++---
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 13 +++++++---
.../TestApplicationHistoryClientService.java | 26 +++++++++++++++++---
...pplicationHistoryManagerOnTimelineStore.java | 12 +++++----
.../hadoop/yarn/server/webapp/AppsBlock.java | 7 +++++-
.../hadoop/yarn/server/webapp/WebServices.java | 17 ++++---------
.../src/site/markdown/TimelineServer.md | 1 +
13 files changed, 83 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 54b33bc..5ce8c63 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -362,6 +362,9 @@ Release 2.8.0 - UNRELEASED
YARN-3581. Deprecate -directlyAccessNodeLabelStore in RMAdminCLI.
(Naganarasimha G R via wangda)
+ YARN-3700. Made generic history service load a number of latest applications
+ according to the parameter or the configuration. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index c84cc76..6892e2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1483,6 +1483,15 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
"0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
+ /**
+ * Defines the max number of applications could be fetched using
+ * REST API or application history protocol and shown in timeline
+ * server web ui.
+ */
+ public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+ APPLICATION_HISTORY_PREFIX + "max-applications";
+ public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+
/** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE =
TIMELINE_SERVICE_PREFIX + "store-class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 1200690..199f4cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -33,6 +33,7 @@ public interface YarnWebParams {
String ENTITY_STRING = "entity.string";
String APP_OWNER = "app.owner";
String APP_STATE = "app.state";
+ String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
String NODE_LABEL = "node.label";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 74a5d1d..c7145b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1404,6 +1404,15 @@
</property>
<property>
+ <description>
+ Defines the max number of applications could be fetched using REST API or
+ application history protocol and shown in timeline server web ui.
+ </description>
+ <name>yarn.timeline-service.generic-application-history.max-applications</name>
+ <value>10000</value>
+ </property>
+
+ <property>
<description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name>
<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index e64ca14..7427926 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -186,7 +186,7 @@ public class ApplicationHistoryClientService extends AbstractService implements
IOException {
GetApplicationsResponse response =
GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
- history.getAllApplications().values()));
+ history.getApplications(request.getLimit()).values()));
return response;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
index 041c31b..c218eec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
@@ -51,16 +51,20 @@ public interface ApplicationHistoryManager {
IOException;
/**
- * This method returns all Application {@link ApplicationReport}s
- *
+ * This method returns the given number of Application
+ * {@link ApplicationReport}s.
+ *
+ * @param appsNum
+ *
* @return map of {@link ApplicationId} to {@link ApplicationReport}s.
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
- Map<ApplicationId, ApplicationReport> getAllApplications()
- throws YarnException, IOException;
+ Map<ApplicationId, ApplicationReport>
+ getApplications(long appsNum) throws YarnException,
+ IOException;
/**
* Application can have multiple application attempts
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index c7cf07b..479858f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -98,7 +98,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws IOException {
Map<ApplicationId, ApplicationHistoryData> histData =
historyStore.getAllApplications();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 2429a68..d3e7b5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -78,6 +79,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
private TimelineDataManager timelineDataManager;
private ApplicationACLsManager aclsManager;
private String serverHttpAddress;
+ private long maxLoadedApplications;
public ApplicationHistoryManagerOnTimelineStore(
TimelineDataManager timelineDataManager,
@@ -91,6 +93,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
protected void serviceInit(Configuration conf) throws Exception {
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
+ maxLoadedApplications =
+ conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
super.serviceInit(conf);
}
@@ -101,12 +106,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws YarnException, IOException {
TimelineEntities entities = timelineDataManager.getEntities(
- ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
- null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
- UserGroupInformation.getLoginUser());
+ ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
+ null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
+ EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps =
new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index d03b26d..ad03718 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -53,6 +53,7 @@ import org.junit.Test;
public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService;
+ private static TimelineDataManager dataManager;
@BeforeClass
public static void setup() throws Exception {
@@ -60,7 +61,7 @@ public class TestApplicationHistoryClientService {
TimelineStore store =
TestApplicationHistoryManagerOnTimelineStore.createStore(2);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
- TimelineDataManager dataManager =
+ dataManager =
new TimelineDataManager(store, aclsManager);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager =
@@ -101,8 +102,27 @@ public class TestApplicationHistoryClientService {
clientService.getApplications(request);
List<ApplicationReport> appReport = response.getApplicationList();
Assert.assertNotNull(appReport);
- Assert.assertEquals(appId, appReport.get(0).getApplicationId());
- Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
+
+ // Create a historyManager, and set the max_apps can be loaded
+ // as 1.
+ Configuration conf = new YarnConfiguration();
+ conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+ ApplicationHistoryManagerOnTimelineStore historyManager2 =
+ new ApplicationHistoryManagerOnTimelineStore(dataManager,
+ new ApplicationACLsManager(conf));
+ historyManager2.init(conf);
+ historyManager2.start();
+ @SuppressWarnings("resource")
+ ApplicationHistoryClientService clientService2 =
+ new ApplicationHistoryClientService(historyManager2);
+ response = clientService2.getApplications(request);
+ appReport = response.getApplicationList();
+ Assert.assertNotNull(appReport);
+ Assert.assertTrue(appReport.size() == 1);
+ // Expected to get the appReport for application with appId1
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index fcdafc2..8672953 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -180,8 +180,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals("test app type", app.getApplicationType());
Assert.assertEquals("user1", app.getUser());
Assert.assertEquals("test queue", app.getQueue());
- Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
- Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 2L
+ + app.getApplicationId().getId(), app.getStartTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 3L
+ + +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
@@ -335,7 +337,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test
public void testGetApplications() throws Exception {
Collection<ApplicationReport> apps =
- historyManager.getAllApplications().values();
+ historyManager.getApplications(Long.MAX_VALUE).values();
Assert.assertNotNull(apps);
Assert.assertEquals(SCALE + 1, apps.size());
ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
@@ -472,12 +474,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
entity.addEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.setEventType(
ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 3L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
"test diagnostics info");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index 5df70c4..b0a3f2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
@@ -70,10 +71,14 @@ public class AppsBlock extends HtmlBlock {
reqAppStates.add(YarnApplicationState.valueOf(stateString.trim()));
}
}
-
callerUGI = getCallerUGI();
final GetApplicationsRequest request =
GetApplicationsRequest.newInstance(reqAppStates);
+ String appsNumStr = $(APPS_NUM);
+ if (appsNumStr != null && !appsNumStr.isEmpty()) {
+ long appsNum = Long.parseLong(appsNumStr);
+ request.setLimit(appsNum);
+ }
if (callerUGI == null) {
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index 909bf1d..8404719 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -72,13 +72,11 @@ public class WebServices {
String startedEnd, String finishBegin, String finishEnd,
Set<String> applicationTypes) {
UserGroupInformation callerUGI = getUser(req);
- long num = 0;
- boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
boolean checkAppStates = false;
- long countNum = 0;
+ long countNum = Long.MAX_VALUE;
// set values suitable in case both of begin/end not specified
long sBegin = 0;
@@ -87,7 +85,6 @@ public class WebServices {
long fEnd = Long.MAX_VALUE;
if (count != null && !count.isEmpty()) {
- checkCount = true;
countNum = Long.parseLong(count);
if (countNum <= 0) {
throw new BadRequestException("limit value must be greater then 0");
@@ -148,19 +145,20 @@ public class WebServices {
AppsInfo allApps = new AppsInfo();
Collection<ApplicationReport> appReports = null;
+ final GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance();
+ request.setLimit(countNum);
try {
if (callerUGI == null) {
// TODO: the request should take the params like what RMWebServices does
// in YARN-1819.
- GetApplicationsRequest request = GetApplicationsRequest.newInstance();
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
appReports = callerUGI.doAs(
new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
@Override
public Collection<ApplicationReport> run() throws Exception {
- return appBaseProt.getApplications(
- GetApplicationsRequest.newInstance()).getApplicationList();
+ return appBaseProt.getApplications(request).getApplicationList();
}
});
}
@@ -169,10 +167,6 @@ public class WebServices {
}
for (ApplicationReport appReport : appReports) {
- if (checkCount && num == countNum) {
- break;
- }
-
if (checkAppStates &&
!appStates.contains(StringUtils.toLowerCase(
appReport.getYarnApplicationState().toString()))) {
@@ -212,7 +206,6 @@ public class WebServices {
AppInfo app = new AppInfo(appReport);
allApps.add(app);
- num++;
}
return allApps;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dd9e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index e177215..5a18fb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -183,6 +183,7 @@ selected if this policy is either of `HTTPS_ONLY` or `HTTP_AND_HTTPS`.
| `yarn.timeline-service.handler-thread-count` | Handler thread count to serve the client RPC requests. Defaults to `10`. |
| `yarn.timeline-service.client.max-retries` | The maximum number of retries for attempts to publish data to the timeline service.Defaults to `30`. |
| `yarn.timeline-service.client.retry-interval-ms` | The interval in milliseconds between retries for the timeline service client. Defaults to `1000`. |
+| `yarn.timeline-service.generic-application-history.max-applications` | The max number of applications could be fetched by using REST API or application history protocol and shown in timeline server web ui. Defaults to `10000`. |
[30/50] [abbrv] hadoop git commit: HDFS-7401. Add block info to
DFSInputStream' WARN message when it adds node to deadNodes (Contributed by
Arshad Mohammad)
Posted by zj...@apache.org.
HDFS-7401. Add block info to DFSInputStream' WARN message when it adds node to deadNodes (Contributed by Arshad Mohammad)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f26da60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f26da60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f26da60
Branch: refs/heads/YARN-2928
Commit: 0f26da602f6f5ae84b4975cb85c2d4f48c4e394b
Parents: fdc6ac8
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri May 29 16:36:19 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:57 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f26da60/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 514076b..23699a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -819,6 +819,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8407. hdfsListDirectory must set errno to 0 on success (Masatake
Iwasaki via Colin P. McCabe)
+ HDFS-7401. Add block info to DFSInputStream' WARN message when it adds
+ node to deadNodes (Arshad Mohammad via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f26da60/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index d1e0b9a..8a3f730 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -631,8 +631,8 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
fetchBlockAt(target);
} else {
connectFailedOnce = true;
- DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block"
- + ", add to deadNodes and continue. " + ex, ex);
+ DFSClient.LOG.warn("Failed to connect to " + targetAddr + " for block "
+ +targetBlock.getBlock()+ ", add to deadNodes and continue. " + ex, ex);
// Put chosen node into dead list, continue
addToDeadNodes(chosenNode);
}
[28/50] [abbrv] hadoop git commit: HADOOP-12022. fix site -Pdocs
-Pdist in hadoop-project-dist; cleanout remaining forrest bits (aw)
Posted by zj...@apache.org.
HADOOP-12022. fix site -Pdocs -Pdist in hadoop-project-dist; cleanout remaining forrest bits (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b17498d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b17498d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b17498d6
Branch: refs/heads/YARN-2928
Commit: b17498d6e188a61cdc89b84df4bfef83152d2ecf
Parents: 44f4df5
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 13:23:49 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
hadoop-common-project/hadoop-common/pom.xml | 1 -
.../src/main/docs/changes/ChangesFancyStyle.css | 170 -
.../main/docs/changes/ChangesSimpleStyle.css | 49 -
.../src/main/docs/changes/changes2html.pl | 286 -
.../src/main/docs/releasenotes.html | 29099 -----------------
.../src/main/docs/src/documentation/README.txt | 7 -
.../classes/CatalogManager.properties | 40 -
.../main/docs/src/documentation/conf/cli.xconf | 327 -
.../src/documentation/content/xdocs/index.xml | 48 -
.../src/documentation/content/xdocs/site.xml | 263 -
.../src/documentation/content/xdocs/tabs.xml | 37 -
.../resources/images/architecture.gif | Bin 15461 -> 0 bytes
.../resources/images/common-logo.jpg | Bin 5887 -> 0 bytes
.../resources/images/core-logo.gif | Bin 6665 -> 0 bytes
.../documentation/resources/images/favicon.ico | Bin 766 -> 0 bytes
.../resources/images/hadoop-logo-big.jpg | Bin 127869 -> 0 bytes
.../resources/images/hadoop-logo.jpg | Bin 9443 -> 0 bytes
.../resources/images/hdfsarchitecture.gif | Bin 17653 -> 0 bytes
.../resources/images/hdfsarchitecture.odg | Bin 41298 -> 0 bytes
.../resources/images/hdfsarchitecture.png | Bin 40571 -> 0 bytes
.../resources/images/hdfsdatanodes.gif | Bin 16060 -> 0 bytes
.../resources/images/hdfsdatanodes.odg | Bin 37296 -> 0 bytes
.../resources/images/hdfsdatanodes.png | Bin 30012 -> 0 bytes
.../main/docs/src/documentation/skinconf.xml | 366 -
.../hadoop-common/src/main/docs/status.xml | 75 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 2 -
hadoop-project-dist/pom.xml | 16 -
28 files changed, 3 insertions(+), 30786 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index aba9087..71b71a0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -477,6 +477,9 @@ Trunk (Unreleased)
HADOOP-11983. HADOOP_USER_CLASSPATH_FIRST works the opposite of what it is
supposed to do (Sangjin Lee via aw)
+ HADOOP-12022. fix site -Pdocs -Pdist in hadoop-project-dist; cleanout
+ remaining forrest bits (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 6f9ecf7..3043889 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -488,7 +488,6 @@
<exclude>CHANGES.txt</exclude>
<exclude>.idea/**</exclude>
<exclude>src/main/conf/*</exclude>
- <exclude>src/main/docs/**</exclude>
<exclude>dev-support/jdiff/**</exclude>
<exclude>src/main/native/*</exclude>
<exclude>src/main/native/config/*</exclude>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesFancyStyle.css
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesFancyStyle.css b/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesFancyStyle.css
deleted file mode 100644
index 5eef241..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesFancyStyle.css
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-/**
- * General
- */
-
-img { border: 0; }
-
-#content table {
- border: 0;
- width: 100%;
-}
-/*Hack to get IE to render the table at 100%*/
-* html #content table { margin-left: -3px; }
-
-#content th,
-#content td {
- margin: 0;
- padding: 0;
- vertical-align: top;
-}
-
-.clearboth {
- clear: both;
-}
-
-.note, .warning, .fixme {
- border: solid black 1px;
- margin: 1em 3em;
-}
-
-.note .label {
- background: #369;
- color: white;
- font-weight: bold;
- padding: 5px 10px;
-}
-.note .content {
- background: #F0F0FF;
- color: black;
- line-height: 120%;
- font-size: 90%;
- padding: 5px 10px;
-}
-.warning .label {
- background: #C00;
- color: white;
- font-weight: bold;
- padding: 5px 10px;
-}
-.warning .content {
- background: #FFF0F0;
- color: black;
- line-height: 120%;
- font-size: 90%;
- padding: 5px 10px;
-}
-.fixme .label {
- background: #C6C600;
- color: black;
- font-weight: bold;
- padding: 5px 10px;
-}
-.fixme .content {
- padding: 5px 10px;
-}
-
-/**
- * Typography
- */
-
-body {
- font-family: verdana, "Trebuchet MS", arial, helvetica, sans-serif;
- font-size: 100%;
-}
-
-#content {
- font-family: Georgia, Palatino, Times, serif;
- font-size: 95%;
-}
-#tabs {
- font-size: 70%;
-}
-#menu {
- font-size: 80%;
-}
-#footer {
- font-size: 70%;
-}
-
-h1, h2, h3, h4, h5, h6 {
- font-family: "Trebuchet MS", verdana, arial, helvetica, sans-serif;
- font-weight: bold;
- margin-top: 1em;
- margin-bottom: .5em;
-}
-
-h1 {
- margin-top: 0;
- margin-bottom: 1em;
- font-size: 1.4em;
- background-color: 73CAFF
-}
-#content h1 {
- font-size: 160%;
- margin-bottom: .5em;
-}
-#menu h1 {
- margin: 0;
- padding: 10px;
- background: #336699;
- color: white;
-}
-h2 {
- font-size: 120%;
- background-color: 73CAFF
-}
-h3 { font-size: 100%; }
-h4 { font-size: 90%; }
-h5 { font-size: 80%; }
-h6 { font-size: 75%; }
-
-p {
- line-height: 120%;
- text-align: left;
- margin-top: .5em;
- margin-bottom: 1em;
-}
-
-#content li,
-#content th,
-#content td,
-#content li ul,
-#content li ol{
- margin-top: .5em;
- margin-bottom: .5em;
-}
-
-
-#content li li,
-#minitoc-area li{
- margin-top: 0em;
- margin-bottom: 0em;
-}
-
-#content .attribution {
- text-align: right;
- font-style: italic;
- font-size: 85%;
- margin-top: 1em;
-}
-
-.codefrag {
- font-family: "Courier New", Courier, monospace;
- font-size: 110%;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesSimpleStyle.css
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesSimpleStyle.css b/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesSimpleStyle.css
deleted file mode 100644
index 407d0f1..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/changes/ChangesSimpleStyle.css
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-body {
- font-family: Courier New, monospace;
- font-size: 10pt;
-}
-
-h1 {
- font-family: Courier New, monospace;
- font-size: 10pt;
-}
-
-h2 {
- font-family: Courier New, monospace;
- font-size: 10pt;
-}
-
-h3 {
- font-family: Courier New, monospace;
- font-size: 10pt;
-}
-
-a:link {
- color: blue;
-}
-
-a:visited {
- color: purple;
-}
-
-li {
- margin-top: 1em;
- margin-bottom: 1em;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl b/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
deleted file mode 100755
index 4431e57..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/changes/changes2html.pl
+++ /dev/null
@@ -1,286 +0,0 @@
-#!/usr/bin/perl
-#
-# Transforms Lucene Java's CHANGES.txt into Changes.html
-#
-# Input is on STDIN, output is to STDOUT
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-use strict;
-use warnings;
-
-my $jira_url_prefix = 'http://issues.apache.org/jira/browse/';
-my $title = undef;
-my $release = undef;
-my $sections = undef;
-my $items = undef;
-my $first_relid = undef;
-my $second_relid = undef;
-my @releases = ();
-
-my @lines = <>; # Get all input at once
-
-#
-# Parse input and build hierarchical release structure in @releases
-#
-for (my $line_num = 0 ; $line_num <= $#lines ; ++$line_num) {
- $_ = $lines[$line_num];
- next unless (/\S/); # Skip blank lines
-
- unless ($title) {
- if (/\S/) {
- s/^\s+//; # Trim leading whitespace
- s/\s+$//; # Trim trailing whitespace
- }
- $title = $_;
- next;
- }
-
- if (/^(Release)|(Trunk)/) { # Release headings
- $release = $_;
- $sections = [];
- push @releases, [ $release, $sections ];
- ($first_relid = lc($release)) =~ s/\s+/_/g if ($#releases == 0);
- ($second_relid = lc($release)) =~ s/\s+/_/g if ($#releases == 1);
- $items = undef;
- next;
- }
-
- # Section heading: 2 leading spaces, words all capitalized
- if (/^ ([A-Z]+)\s*/) {
- my $heading = $_;
- $items = [];
- push @$sections, [ $heading, $items ];
- next;
- }
-
- # Handle earlier releases without sections - create a headless section
- unless ($items) {
- $items = [];
- push @$sections, [ undef, $items ];
- }
-
- my $type;
- if (@$items) { # A list item has been encountered in this section before
- $type = $items->[0]; # 0th position of items array is list type
- } else {
- $type = get_list_type($_);
- push @$items, $type;
- }
-
- if ($type eq 'numbered') { # The modern items list style
- # List item boundary is another numbered item or an unindented line
- my $line;
- my $item = $_;
- $item =~ s/^(\s{0,2}\d+\.\s*)//; # Trim the leading item number
- my $leading_ws_width = length($1);
- $item =~ s/\s+$//; # Trim trailing whitespace
- $item .= "\n";
-
- while ($line_num < $#lines
- and ($line = $lines[++$line_num]) !~ /^(?:\s{0,2}\d+\.\s*\S|\S)/) {
- $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace
- $line =~ s/\s+$//; # Trim trailing whitespace
- $item .= "$line\n";
- }
- $item =~ s/\n+\Z/\n/; # Trim trailing blank lines
- push @$items, $item;
- --$line_num unless ($line_num == $#lines);
- } elsif ($type eq 'paragraph') { # List item boundary is a blank line
- my $line;
- my $item = $_;
- $item =~ s/^(\s+)//;
- my $leading_ws_width = defined($1) ? length($1) : 0;
- $item =~ s/\s+$//; # Trim trailing whitespace
- $item .= "\n";
-
- while ($line_num < $#lines and ($line = $lines[++$line_num]) =~ /\S/) {
- $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace
- $line =~ s/\s+$//; # Trim trailing whitespace
- $item .= "$line\n";
- }
- push @$items, $item;
- --$line_num unless ($line_num == $#lines);
- } else { # $type is one of the bulleted types
- # List item boundary is another bullet or a blank line
- my $line;
- my $item = $_;
- $item =~ s/^(\s*$type\s*)//; # Trim the leading bullet
- my $leading_ws_width = length($1);
- $item =~ s/\s+$//; # Trim trailing whitespace
- $item .= "\n";
-
- while ($line_num < $#lines
- and ($line = $lines[++$line_num]) !~ /^\s*(?:$type|\Z)/) {
- $line =~ s/^\s{$leading_ws_width}//; # Trim leading whitespace
- $line =~ s/\s+$//; # Trim trailing whitespace
- $item .= "$line\n";
- }
- push @$items, $item;
- --$line_num unless ($line_num == $#lines);
- }
-}
-
-#
-# Print HTML-ified version to STDOUT
-#
-print<<"__HTML_HEADER__";
-<!--
-**********************************************************
-** WARNING: This file is generated from CHANGES.txt by the
-** Perl script 'changes2html.pl'.
-** Do *not* edit this file!
-**********************************************************
-
-****************************************************************************
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-****************************************************************************
--->
-<html>
-<head>
- <title>$title</title>
- <link rel="stylesheet" href="ChangesFancyStyle.css" title="Fancy">
- <link rel="alternate stylesheet" href="ChangesSimpleStyle.css" title="Simple">
- <META http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
- <SCRIPT>
- function toggleList(e) {
- element = document.getElementById(e).style;
- element.display == 'none' ? element.display = 'block' : element.display='none';
- }
- function collapse() {
- for (var i = 0; i < document.getElementsByTagName("ul").length; i++) {
- var list = document.getElementsByTagName("ul")[i];
- if (list.id != '$first_relid' && list.id != '$second_relid') {
- list.style.display = "none";
- }
- }
- for (var i = 0; i < document.getElementsByTagName("ol").length; i++) {
- document.getElementsByTagName("ol")[i].style.display = "none";
- }
- }
- window.onload = collapse;
- </SCRIPT>
-</head>
-<body>
-
-<a href="http://hadoop.apache.org/core/"><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Scalable Computing Platform"></a>
-<h1>$title</h1>
-
-__HTML_HEADER__
-
-my $heading;
-my $relcnt = 0;
-my $header = 'h2';
-for my $rel (@releases) {
- if (++$relcnt == 3) {
- $header = 'h3';
- print "<h2><a href=\"javascript:toggleList('older')\">";
- print "Older Releases";
- print "</a></h2>\n";
- print "<ul id=\"older\">\n"
- }
-
- ($release, $sections) = @$rel;
-
- # The first section heading is undefined for the older sectionless releases
- my $has_release_sections = $sections->[0][0];
-
- (my $relid = lc($release)) =~ s/\s+/_/g;
- print "<$header><a href=\"javascript:toggleList('$relid')\">";
- print "$release";
- print "</a></$header>\n";
- print "<ul id=\"$relid\">\n"
- if ($has_release_sections);
-
- for my $section (@$sections) {
- ($heading, $items) = @$section;
- (my $sectid = lc($heading)) =~ s/\s+/_/g;
- my $numItemsStr = $#{$items} > 0 ? "($#{$items})" : "(none)";
-
- print " <li><a href=\"javascript:toggleList('$relid.$sectid')\">",
- ($heading || ''), "</a> $numItemsStr\n"
- if ($has_release_sections);
-
- my $list_type = $items->[0] || '';
- my $list = ($has_release_sections || $list_type eq 'numbered' ? 'ol' : 'ul');
- my $listid = $sectid ? "$relid.$sectid" : $relid;
- print " <$list id=\"$listid\">\n";
-
- for my $itemnum (1..$#{$items}) {
- my $item = $items->[$itemnum];
- $item =~ s:&:&:g; # Escape HTML metachars
- $item =~ s:<:<:g;
- $item =~ s:>:>:g;
-
- $item =~ s:\s*(\([^)"]+?\))\s*$:<br />$1:; # Separate attribution
- $item =~ s:\n{2,}:\n<p/>\n:g; # Keep paragraph breaks
- $item =~ s{(?:${jira_url_prefix})?(HADOOP-\d+)} # Link to JIRA Common
- {<a href="${jira_url_prefix}$1">$1</a>}g;
- $item =~ s{(?:${jira_url_prefix})?(HDFS-\d+)} # Link to JIRA Hdfs
- {<a href="${jira_url_prefix}$1">$1</a>}g;
- $item =~ s{(?:${jira_url_prefix})?(MAPREDUCE-\d+)} # Link to JIRA MR
- {<a href="${jira_url_prefix}$1">$1</a>}g;
- print " <li>$item</li>\n";
- }
- print " </$list>\n";
- print " </li>\n" if ($has_release_sections);
- }
- print "</ul>\n" if ($has_release_sections);
-}
-print "</ul>\n" if ($relcnt > 3);
-print "</body>\n</html>\n";
-
-
-#
-# Subroutine: get_list_type
-#
-# Takes one parameter:
-#
-# - The first line of a sub-section/point
-#
-# Returns one scalar:
-#
-# - The list type: 'numbered'; or one of the bulleted types '-', or '.' or
-# 'paragraph'.
-#
-sub get_list_type {
- my $first_list_item_line = shift;
- my $type = 'paragraph'; # Default to paragraph type
-
- if ($first_list_item_line =~ /^\s{0,2}\d+\.\s+\S+/) {
- $type = 'numbered';
- } elsif ($first_list_item_line =~ /^\s*([-.])\s+\S+/) {
- $type = $1;
- }
- return $type;
-}
-
-1;
[08/50] [abbrv] hadoop git commit: HDFS-8431. hdfs crypto class not
found in Windows. Contributed by Anu Engineer.
Posted by zj...@apache.org.
HDFS-8431. hdfs crypto class not found in Windows. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b39cff2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b39cff2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b39cff2
Branch: refs/heads/YARN-2928
Commit: 2b39cff2e8545ad4756d4ead42e52661be7bfc22
Parents: 81dd9e8
Author: cnauroth <cn...@apache.org>
Authored: Wed May 27 22:54:00 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:53 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd | 7 ++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b39cff2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 439b53a..47481a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -810,6 +810,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8268. Port conflict log for data node server is not sufficient
(Mohammad Shahid Khan via vinayakumarb)
+ HDFS-8431. hdfs crypto class not found in Windows.
+ (Anu Engineer via cnauroth)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b39cff2/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
index 7883252..8115349 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
@@ -59,7 +59,7 @@ if "%1" == "--loglevel" (
)
)
- set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath
+ set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto
for %%i in ( %hdfscommands% ) do (
if %hdfs-command% == %%i set hdfscommand=true
)
@@ -175,6 +175,10 @@ goto :eof
set CLASS=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
goto :eof
+:crypto
+ set CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
+ goto :eof
+
@rem This changes %1, %2 etc. Hence those cannot be used after calling this.
:make_command_arguments
if "%1" == "--config" (
@@ -227,6 +231,7 @@ goto :eof
@echo lsSnapshottableDir list all snapshottable dirs owned by the current user
@echo Use -help to see options
@echo cacheadmin configure the HDFS cache
+ @echo crypto configure HDFS encryption zones
@echo mover run a utility to move block replicas across storage types
@echo storagepolicies list/get/set block storage policies
@echo.
[43/50] [abbrv] hadoop git commit: YARN-2900. Application (Attempt
and Container) Not Found in AHS results in Internal Server Error (500).
Contributed by Zhijie Shen and Mit Desai
Posted by zj...@apache.org.
YARN-2900. Application (Attempt and Container) Not Found in AHS results
in Internal Server Error (500). Contributed by Zhijie Shen and Mit Desai
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a977bed5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a977bed5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a977bed5
Branch: refs/heads/YARN-2928
Commit: a977bed50936c7054f270f93569fa0f53002fe0b
Parents: 8bc36be
Author: Xuan <xg...@apache.org>
Authored: Sun May 31 15:37:05 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:13:00 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../TestApplicationHistoryClientService.java | 70 +++++++++++++++++++-
.../webapp/TestAHSWebServices.java | 63 +++++++++++++++++-
.../hadoop/yarn/server/webapp/WebServices.java | 27 +++++---
4 files changed, 151 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a977bed5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6f5faa9..e770a5b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -694,6 +694,9 @@ Release 2.7.1 - UNRELEASED
YARN-3686. CapacityScheduler should trim default_node_label_expression.
(Sunil G via wangda)
+ YARN-2900. Application (Attempt and Container) Not Found in AHS results
+ in Internal Server Error (500). (Zhijie Shen and Mit Desai via xgong)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a977bed5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 764ecb9..d04903c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -41,6 +41,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
@@ -54,12 +57,13 @@ public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService;
private static TimelineDataManager dataManager;
+ private final static int MAX_APPS = 2;
@BeforeClass
public static void setup() throws Exception {
Configuration conf = new YarnConfiguration();
TimelineStore store =
- TestApplicationHistoryManagerOnTimelineStore.createStore(2);
+ TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
dataManager =
new TimelineDataManager(store, aclsManager);
@@ -72,6 +76,70 @@ public class TestApplicationHistoryClientService {
}
@Test
+ public void testApplicationNotFound() throws IOException, YarnException {
+ ApplicationId appId = null;
+ appId = ApplicationId.newInstance(0, MAX_APPS + 1);
+ GetApplicationReportRequest request =
+ GetApplicationReportRequest.newInstance(appId);
+ try {
+ @SuppressWarnings("unused")
+ GetApplicationReportResponse response =
+ clientService.getApplicationReport(request);
+ Assert.fail("Exception should have been thrown before we reach here.");
+ } catch (ApplicationNotFoundException e) {
+ //This exception is expected.
+ Assert.assertTrue(e.getMessage().contains(
+ "doesn't exist in the timeline store"));
+ } catch (Exception e) {
+ Assert.fail("Undesired exception caught");
+ }
+ }
+
+ @Test
+ public void testApplicationAttemptNotFound() throws IOException, YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
+ GetApplicationAttemptReportRequest request =
+ GetApplicationAttemptReportRequest.newInstance(appAttemptId);
+ try {
+ @SuppressWarnings("unused")
+ GetApplicationAttemptReportResponse response =
+ clientService.getApplicationAttemptReport(request);
+ Assert.fail("Exception should have been thrown before we reach here.");
+ } catch (ApplicationAttemptNotFoundException e) {
+ //This Exception is expected
+ System.out.println(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains(
+ "doesn't exist in the timeline store"));
+ } catch (Exception e) {
+ Assert.fail("Undesired exception caught");
+ }
+ }
+
+ @Test
+ public void testContainerNotFound() throws IOException, YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId = ContainerId.newContainerId(appAttemptId,
+ MAX_APPS + 1);
+ GetContainerReportRequest request =
+ GetContainerReportRequest.newInstance(containerId);
+ try {
+ @SuppressWarnings("unused")
+ GetContainerReportResponse response =
+ clientService.getContainerReport(request);
+ } catch (ContainerNotFoundException e) {
+ //This exception is expected
+ Assert.assertTrue(e.getMessage().contains(
+ "doesn't exist in the timeline store"));
+ } catch (Exception e) {
+ Assert.fail("Undesired exception caught");
+ }
+ }
+
+ @Test
public void testApplicationReport() throws IOException, YarnException {
ApplicationId appId = null;
appId = ApplicationId.newInstance(0, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a977bed5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index 613df72..b5ff6a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -82,12 +82,13 @@ public class TestAHSWebServices extends JerseyTestBase {
private static ApplicationHistoryClientService historyClientService;
private static final String[] USERS = new String[] { "foo" , "bar" };
+ private static final int MAX_APPS = 5;
@BeforeClass
public static void setupClass() throws Exception {
Configuration conf = new YarnConfiguration();
TimelineStore store =
- TestApplicationHistoryManagerOnTimelineStore.createStore(5);
+ TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
TimelineDataManager dataManager =
new TimelineDataManager(store, aclsManager);
@@ -165,6 +166,66 @@ public class TestAHSWebServices extends JerseyTestBase {
}
@Test
+ public void testInvalidApp() {
+ ApplicationId appId = ApplicationId.newInstance(0, MAX_APPS + 1);
+ WebResource r = resource();
+ ClientResponse response =
+ r.path("ws").path("v1").path("applicationhistory").path("apps")
+ .path(appId.toString())
+ .queryParam("user.name", USERS[round])
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ assertEquals("404 not found expected", Status.NOT_FOUND,
+ response.getClientResponseStatus());
+ }
+
+ @Test
+ public void testInvalidAttempt() {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
+ WebResource r = resource();
+ ClientResponse response =
+ r.path("ws").path("v1").path("applicationhistory").path("apps")
+ .path(appId.toString()).path("appattempts")
+ .path(appAttemptId.toString())
+ .queryParam("user.name", USERS[round])
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ if (round == 1) {
+ assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
+ return;
+ }
+ assertEquals("404 not found expected", Status.NOT_FOUND,
+ response.getClientResponseStatus());
+ }
+
+ @Test
+ public void testInvalidContainer() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId = ContainerId.newContainerId(appAttemptId,
+ MAX_APPS + 1);
+ WebResource r = resource();
+ ClientResponse response =
+ r.path("ws").path("v1").path("applicationhistory").path("apps")
+ .path(appId.toString()).path("appattempts")
+ .path(appAttemptId.toString()).path("containers")
+ .path(containerId.toString())
+ .queryParam("user.name", USERS[round])
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ if (round == 1) {
+ assertEquals(
+ Status.FORBIDDEN, response.getClientResponseStatus());
+ return;
+ }
+ assertEquals("404 not found expected", Status.NOT_FOUND,
+ response.getClientResponseStatus());
+ }
+
+ @Test
public void testInvalidUri() throws JSONException, Exception {
WebResource r = resource();
String responseStr = "";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a977bed5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index 8404719..3064145 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -47,6 +47,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@@ -477,17 +480,21 @@ public class WebServices {
private static void rewrapAndThrowException(Exception e) {
if (e instanceof UndeclaredThrowableException) {
- if (e.getCause() instanceof AuthorizationException) {
- throw new ForbiddenException(e.getCause());
- } else {
- throw new WebApplicationException(e.getCause());
- }
+ rewrapAndThrowThrowable(e.getCause());
} else {
- if (e instanceof AuthorizationException) {
- throw new ForbiddenException(e);
- } else {
- throw new WebApplicationException(e);
- }
+ rewrapAndThrowThrowable(e);
+ }
+ }
+
+ private static void rewrapAndThrowThrowable(Throwable t) {
+ if (t instanceof AuthorizationException) {
+ throw new ForbiddenException(t);
+ } else if (t instanceof ApplicationNotFoundException ||
+ t instanceof ApplicationAttemptNotFoundException ||
+ t instanceof ContainerNotFoundException) {
+ throw new NotFoundException(t);
+ } else {
+ throw new WebApplicationException(t);
}
}
[26/50] [abbrv] hadoop git commit: HADOOP-12022. fix site -Pdocs
-Pdist in hadoop-project-dist; cleanout remaining forrest bits (aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/README.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/README.txt b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/README.txt
deleted file mode 100644
index 9bc261b..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/README.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-This is the base documentation directory.
-
-skinconf.xml # This file customizes Forrest for your project. In it, you
- # tell forrest the project name, logo, copyright info, etc
-
-sitemap.xmap # Optional. This sitemap is consulted before all core sitemaps.
- # See http://forrest.apache.org/docs/project-sitemap.html
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/classes/CatalogManager.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/classes/CatalogManager.properties b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/classes/CatalogManager.properties
deleted file mode 100644
index b9cb584..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/classes/CatalogManager.properties
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#=======================================================================
-# CatalogManager.properties
-#
-# This is the default properties file for Apache Forrest.
-# This facilitates local configuration of application-specific catalogs.
-#
-# See the Apache Forrest documentation:
-# http://forrest.apache.org/docs/your-project.html
-# http://forrest.apache.org/docs/validation.html
-
-# verbosity ... level of messages for status/debug
-# See forrest/src/core/context/WEB-INF/cocoon.xconf
-
-# catalogs ... list of additional catalogs to load
-# (Note that Apache Forrest will automatically load its own default catalog
-# from src/core/context/resources/schema/catalog.xcat)
-# use full pathnames
-# pathname separator is always semi-colon (;) regardless of operating system
-# directory separator is always slash (/) regardless of operating system
-#
-#catalogs=/home/me/forrest/my-site/src/documentation/resources/schema/catalog.xcat
-catalogs=
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/conf/cli.xconf
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/conf/cli.xconf b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/conf/cli.xconf
deleted file mode 100644
index 5c6e245..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/conf/cli.xconf
+++ /dev/null
@@ -1,327 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!--+
- | This is the Apache Cocoon command line configuration file.
- | Here you give the command line interface details of where
- | to find various aspects of your Cocoon installation.
- |
- | If you wish, you can also use this file to specify the URIs
- | that you wish to generate.
- |
- | The current configuration information in this file is for
- | building the Cocoon documentation. Therefore, all links here
- | are relative to the build context dir, which, in the build.xml
- | file, is set to ${build.context}
- |
- | Options:
- | verbose: increase amount of information presented
- | to standard output (default: false)
- | follow-links: whether linked pages should also be
- | generated (default: true)
- | precompile-only: precompile sitemaps and XSP pages, but
- | do not generate any pages (default: false)
- | confirm-extensions: check the mime type for the generated page
- | and adjust filename and links extensions
- | to match the mime type
- | (e.g. text/html->.html)
- |
- | Note: Whilst using an xconf file to configure the Cocoon
- | Command Line gives access to more features, the use of
- | command line parameters is more stable, as there are
- | currently plans to improve the xconf format to allow
- | greater flexibility. If you require a stable and
- | consistent method for accessing the CLI, it is recommended
- | that you use the command line parameters to configure
- | the CLI. See documentation at:
- | http://cocoon.apache.org/2.1/userdocs/offline/
- | http://wiki.apache.org/cocoon/CommandLine
- |
- +-->
-
-<cocoon verbose="true"
- follow-links="true"
- precompile-only="false"
- confirm-extensions="false">
-
- <!--+
- | The context directory is usually the webapp directory
- | containing the sitemap.xmap file.
- |
- | The config file is the cocoon.xconf file.
- |
- | The work directory is used by Cocoon to store temporary
- | files and cache files.
- |
- | The destination directory is where generated pages will
- | be written (assuming the 'simple' mapper is used, see
- | below)
- +-->
- <context-dir>.</context-dir>
- <config-file>WEB-INF/cocoon.xconf</config-file>
- <work-dir>../tmp/cocoon-work</work-dir>
- <dest-dir>../site</dest-dir>
-
- <!--+
- | A checksum file can be used to store checksums for pages
- | as they are generated. When the site is next generated,
- | files will not be written if their checksum has not changed.
- | This means that it will be easier to detect which files
- | need to be uploaded to a server, using the timestamp.
- |
- | The default path is relative to the core webapp directory.
- | An asolute path can be used.
- +-->
- <!-- <checksums-uri>build/work/checksums</checksums-uri>-->
-
- <!--+
- | Broken link reporting options:
- | Report into a text file, one link per line:
- | <broken-links type="text" report="filename"/>
- | Report into an XML file:
- | <broken-links type="xml" report="filename"/>
- | Ignore broken links (default):
- | <broken-links type="none"/>
- |
- | Two attributes to this node specify whether a page should
- | be generated when an error has occured. 'generate' specifies
- | whether a page should be generated (default: true) and
- | extension specifies an extension that should be appended
- | to the generated page's filename (default: none)
- |
- | Using this, a quick scan through the destination directory
- | will show broken links, by their filename extension.
- +-->
- <broken-links type="xml"
- file="../brokenlinks.xml"
- generate="false"
- extension=".error"
- show-referrers="true"/>
-
- <!--+
- | Load classes at startup. This is necessary for generating
- | from sites that use SQL databases and JDBC.
- | The <load-class> element can be repeated if multiple classes
- | are needed.
- +-->
- <!--
- <load-class>org.firebirdsql.jdbc.Driver</load-class>
- -->
-
- <!--+
- | Configures logging.
- | The 'log-kit' parameter specifies the location of the log kit
- | configuration file (usually called logkit.xconf.
- |
- | Logger specifies the logging category (for all logging prior
- | to other Cocoon logging categories taking over)
- |
- | Available log levels are:
- | DEBUG: prints all level of log messages.
- | INFO: prints all level of log messages except DEBUG
- | ones.
- | WARN: prints all level of log messages except DEBUG
- | and INFO ones.
- | ERROR: prints all level of log messages except DEBUG,
- | INFO and WARN ones.
- | FATAL_ERROR: prints only log messages of this level
- +-->
- <!-- <logging log-kit="WEB-INF/logkit.xconf" logger="cli" level="ERROR" /> -->
-
- <!--+
- | Specifies the filename to be appended to URIs that
- | refer to a directory (i.e. end with a forward slash).
- +-->
- <default-filename>index.html</default-filename>
-
- <!--+
- | Specifies a user agent string to the sitemap when
- | generating the site.
- |
- | A generic term for a web browser is "user agent". Any
- | user agent, when connecting to a web server, will provide
- | a string to identify itself (e.g. as Internet Explorer or
- | Mozilla). It is possible to have Cocoon serve different
- | content depending upon the user agent string provided by
- | the browser. If your site does this, then you may want to
- | use this <user-agent> entry to provide a 'fake' user agent
- | to Cocoon, so that it generates the correct version of your
- | site.
- |
- | For most sites, this can be ignored.
- +-->
- <!--
- <user-agent>Cocoon Command Line Environment 2.1</user-agent>
- -->
-
- <!--+
- | Specifies an accept string to the sitemap when generating
- | the site.
- | User agents can specify to an HTTP server what types of content
- | (by mime-type) they are able to receive. E.g. a browser may be
- | able to handle jpegs, but not pngs. The HTTP accept header
- | allows the server to take the browser's capabilities into account,
- | and only send back content that it can handle.
- |
- | For most sites, this can be ignored.
- +-->
-
- <accept>*/*</accept>
-
- <!--+
- | Specifies which URIs should be included or excluded, according
- | to wildcard patterns.
- |
- | These includes/excludes are only relevant when you are following
- | links. A link URI must match an include pattern (if one is given)
- | and not match an exclude pattern, if it is to be followed by
- | Cocoon. It can be useful, for example, where there are links in
- | your site to pages that are not generated by Cocoon, such as
- | references to api-documentation.
- |
- | By default, all URIs are included. If both include and exclude
- | patterns are specified, a URI is first checked against the
- | include patterns, and then against the exclude patterns.
- |
- | Multiple patterns can be given, using muliple include or exclude
- | nodes.
- |
- | The order of the elements is not significant, as only the first
- | successful match of each category is used.
- |
- | Currently, only the complete source URI can be matched (including
- | any URI prefix). Future plans include destination URI matching
- | and regexp matching. If you have requirements for these, contact
- | dev@cocoon.apache.org.
- +-->
-
- <exclude pattern="**/"/>
- <exclude pattern="api/**"/>
- <exclude pattern="jdiff/**"/>
- <exclude pattern="changes.html"/>
- <exclude pattern="releasenotes.html"/>
-
-<!--
- This is a workaround for FOR-284 "link rewriting broken when
- linking to xml source views which contain site: links".
- See the explanation there and in declare-broken-site-links.xsl
--->
- <exclude pattern="site:**"/>
- <exclude pattern="ext:**"/>
- <exclude pattern="lm:**"/>
- <exclude pattern="**/site:**"/>
- <exclude pattern="**/ext:**"/>
- <exclude pattern="**/lm:**"/>
-
- <!-- Exclude tokens used in URLs to ASF mirrors (interpreted by a CGI) -->
- <exclude pattern="[preferred]/**"/>
- <exclude pattern="[location]"/>
-
- <!-- <include-links extension=".html"/>-->
-
- <!--+
- | <uri> nodes specify the URIs that should be generated, and
- | where required, what should be done with the generated pages.
- | They describe the way the URI of the generated file is created
- | from the source page's URI. There are three ways that a generated
- | file URI can be created: append, replace and insert.
- |
- | The "type" attribute specifies one of (append|replace|insert):
- |
- | append:
- | Append the generated page's URI to the end of the source URI:
- |
- | <uri type="append" src-prefix="documents/" src="index.html"
- | dest="build/dest/"/>
- |
- | This means that
- | (1) the "documents/index.html" page is generated
- | (2) the file will be written to "build/dest/documents/index.html"
- |
- | replace:
- | Completely ignore the generated page's URI - just
- | use the destination URI:
- |
- | <uri type="replace" src-prefix="documents/" src="index.html"
- | dest="build/dest/docs.html"/>
- |
- | This means that
- | (1) the "documents/index.html" page is generated
- | (2) the result is written to "build/dest/docs.html"
- | (3) this works only for "single" pages - and not when links
- | are followed
- |
- | insert:
- | Insert generated page's URI into the destination
- | URI at the point marked with a * (example uses fictional
- | zip protocol)
- |
- | <uri type="insert" src-prefix="documents/" src="index.html"
- | dest="zip://*.zip/page.html"/>
- |
- | This means that
- | (1)
- |
- | In any of these scenarios, if the dest attribute is omitted,
- | the value provided globally using the <dest-dir> node will
- | be used instead.
- +-->
- <!--
- <uri type="replace"
- src-prefix="samples/"
- src="hello-world/hello.html"
- dest="build/dest/hello-world.html"/>
- -->
-
- <!--+
- | <uri> nodes can be grouped together in a <uris> node. This
- | enables a group of URIs to share properties. The following
- | properties can be set for a group of URIs:
- | * follow-links: should pages be crawled for links
- | * confirm-extensions: should file extensions be checked
- | for the correct mime type
- | * src-prefix: all source URIs should be
- | pre-pended with this prefix before
- | generation. The prefix is not
- | included when calculating the
- | destination URI
- | * dest: the base destination URI to be
- | shared by all pages in this group
- | * type: the method to be used to calculate
- | the destination URI. See above
- | section on <uri> node for details.
- |
- | Each <uris> node can have a name attribute. When a name
- | attribute has been specified, the -n switch on the command
- | line can be used to tell Cocoon to only process the URIs
- | within this URI group. When no -n switch is given, all
- | <uris> nodes are processed. Thus, one xconf file can be
- | used to manage multiple sites.
- +-->
- <!--
- <uris name="mirrors" follow-links="false">
- <uri type="append" src="mirrors.html"/>
- </uris>
- -->
-
- <!--+
- | File containing URIs (plain text, one per line).
- +-->
- <!--
- <uri-file>uris.txt</uri-file>
- -->
-</cocoon>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/index.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/index.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/index.xml
deleted file mode 100644
index e0df9c9..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/index.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
-
-<document>
-
- <header>
- <title>Overview</title>
- </header>
-
- <body>
- <p>
-The Hadoop Common Documentation describes the common utilities and libraries that support the other Hadoop subprojects.
- </p>
- <p>
-The Hadoop Common Documentation also includes the information you need to get started using Hadoop.
-Begin with the Hadoop <a href="single_node_setup.html">Single Node Setup</a> which shows you how to set up a single-node Hadoop installation.
-Then move on to the Hadoop <a href="cluster_setup.html">Cluster Setup</a> to learn how to set up a multi-node Hadoop installation.
-</p>
- <p>
- Cluster environments commonly work in tandem with MapReduce applications and distributed file systems.
- For information about MapReduce see the
- <a href="http://hadoop.apache.org/mapreduce/docs/current/index.html">MapReduce Documentation</a>.
- For information about the Hadoop Distributed File System (HDFS) see the
- <a href="http://hadoop.apache.org/hdfs/docs/current/index.html">HDFS Documentation</a>.
- </p>
-<p>
-If you have more questions, you can ask on the <a href="ext:lists">Hadoop Common Mailing Lists</a> or browse the <a href="ext:archive">Mailing List Archives</a>.
-</p>
- </body>
-
-</document>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml
deleted file mode 100644
index f9ce3be..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml
+++ /dev/null
@@ -1,263 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
-Forrest site.xml
-
-This file contains an outline of the site's information content. It is used to:
-- Generate the website menus (though these can be overridden - see docs)
-- Provide semantic, location-independent aliases for internal 'site:' URIs, eg
-<link href="site:changes"> links to changes.html (or ../changes.html if in
- subdir).
-- Provide aliases for external URLs in the external-refs section. Eg, <link
- href="ext:cocoon"> links to http://xml.apache.org/cocoon/
-
-See http://forrest.apache.org/docs/linking.html for more info.
--->
-
-<site label="Hadoop" href="" xmlns="http://apache.org/forrest/linkmap/1.0">
-
- <docs label="Getting Started">
- <overview label="Overview" href="index.html" />
- <quickstart label="Single Node Setup" href="single_node_setup.html" />
- <setup label="Cluster Setup" href="cluster_setup.html" />
- </docs>
-
- <docs label="Guides">
- <deployment label="Deployment Layout" href="deployment_layout.html" />
- <commands_manual label="Hadoop Commands" href="commands_manual.html" />
- <fsshell label="File System Shell" href="file_system_shell.html" />
- <SLA label="Service Level Authorization" href="service_level_auth.html"/>
- <native_lib label="Native Libraries" href="native_libraries.html" />
- <superusers label="Superusers Acting On Behalf Of Other Users" href="Superusers.html"/>
- <http_authentication label="Authentication for Hadoop HTTP web-consoles" href="HttpAuthentication.html"/>
- </docs>
-
- <docs label="Miscellaneous">
- <api label="API Docs" href="ext:api/index" />
- <jdiff label="API Changes" href="ext:jdiff/changes" />
- <wiki label="Wiki" href="ext:wiki" />
- <faq label="FAQ" href="ext:faq" />
- <relnotes label="Release Notes" href="ext:relnotes" />
- <changes label="Change Log" href="ext:changes" />
- </docs>
-
- <external-refs>
- <site href="http://hadoop.apache.org/common/"/>
- <lists href="http://hadoop.apache.org/common/mailing_lists.html"/>
- <archive href="http://mail-archives.apache.org/mod_mbox/hadoop-common-commits/"/>
- <releases href="http://hadoop.apache.org/common/releases.html">
- <download href="#Download" />
- </releases>
- <jira href="http://hadoop.apache.org/common/issue_tracking.html"/>
- <wiki href="http://wiki.apache.org/hadoop/Common" />
- <faq href="http://wiki.apache.org/hadoop/Common/FAQ" />
-
- <core-default href="http://hadoop.apache.org/common/docs/current/core-default.html" />
- <hdfs-default href="http://hadoop.apache.org/hdfs/docs/current/hdfs-default.html" />
- <mapred-default href="http://hadoop.apache.org/mapreduce/docs/current/mapred-default.html" />
-
- <mapred-queues href="http://hadoop.apache.org/mapreduce/docs/current/mapred_queues.xml" />
- <capacity-scheduler href="http://hadoop.apache.org/mapreduce/docs/current/capacity_scheduler.html">
- <MemoryBasedTaskScheduling href="#Scheduling+Tasks+Considering+Memory+Requirements" />
- </capacity-scheduler>
- <mapred-tutorial href="http://hadoop.apache.org/mapreduce/docs/current/mapred_tutorial.html" >
- <JobAuthorization href="#Job+Authorization" />
- <ConfiguringMemoryRequirements href="#Configuring+Memory+Requirements+For+A+Job" />
- </mapred-tutorial>
- <streaming href="http://hadoop.apache.org/mapreduce/docs/current/streaming.html" />
- <distcp href="http://hadoop.apache.org/mapreduce/docs/current/distcp.html" />
- <hadoop-archives href="http://hadoop.apache.org/mapreduce/docs/current/hadoop_archives.html" />
-
- <zlib href="http://www.zlib.net/" />
- <gzip href="http://www.gzip.org/" />
- <bzip href="http://www.bzip.org/" />
- <osx href="http://www.apple.com/macosx" />
-
- <relnotes href="releasenotes.html" />
- <changes href="changes.html" />
- <jdiff href="jdiff/">
- <changes href="changes.html" />
- </jdiff>
- <api href="api/">
- <index href="index.html" />
- <org href="org/">
- <apache href="apache/">
- <hadoop href="hadoop/">
- <conf href="conf/">
- <configuration href="Configuration.html">
- <final_parameters href="#FinalParams" />
- <get href="#get(java.lang.String, java.lang.String)" />
- <set href="#set(java.lang.String, java.lang.String)" />
- </configuration>
- </conf>
- <filecache href="filecache/">
- <distributedcache href="DistributedCache.html">
- <addarchivetoclasspath href="#addArchiveToClassPath(org.apache.hadoop.fs.Path,%20org.apache.hadoop.conf.Configuration)" />
- <addfiletoclasspath href="#addFileToClassPath(org.apache.hadoop.fs.Path,%20org.apache.hadoop.conf.Configuration)" />
- <addcachefile href="#addCacheFile(java.net.URI,%20org.apache.hadoop.conf.Configuration)" />
- <addcachearchive href="#addCacheArchive(java.net.URI,%20org.apache.hadoop.conf.Configuration)" />
- <setcachefiles href="#setCacheFiles(java.net.URI[],%20org.apache.hadoop.conf.Configuration)" />
- <setcachearchives href="#setCacheArchives(java.net.URI[],%20org.apache.hadoop.conf.Configuration)" />
- <createsymlink href="#createSymlink(org.apache.hadoop.conf.Configuration)" />
- </distributedcache>
- </filecache>
- <fs href="fs/">
- <filesystem href="FileSystem.html" />
- </fs>
- <io href="io/">
- <closeable href="Closeable.html">
- <close href="#close()" />
- </closeable>
- <sequencefile href="SequenceFile.html" />
- <sequencefilecompressiontype href="SequenceFile.CompressionType.html">
- <none href="#NONE" />
- <record href="#RECORD" />
- <block href="#BLOCK" />
- </sequencefilecompressiontype>
- <writable href="Writable.html" />
- <writablecomparable href="WritableComparable.html" />
- <compress href="compress/">
- <compressioncodec href="CompressionCodec.html" />
- </compress>
- </io>
- <mapred href="mapred/">
- <clusterstatus href="ClusterStatus.html" />
- <counters href="Counters.html" />
- <fileinputformat href="FileInputFormat.html">
- <setinputpaths href="#setInputPaths(org.apache.hadoop.mapred.JobConf,%20org.apache.hadoop.fs.Path[])" />
- <addinputpath href="#addInputPath(org.apache.hadoop.mapred.JobConf,%20org.apache.hadoop.fs.Path)" />
- <setinputpathstring href="#setInputPaths(org.apache.hadoop.mapred.JobConf,%20java.lang.String)" />
- <addinputpathstring href="#addInputPath(org.apache.hadoop.mapred.JobConf,%20java.lang.String)" />
- </fileinputformat>
- <fileoutputformat href="FileOutputFormat.html">
- <getoutputpath href="#getOutputPath(org.apache.hadoop.mapred.JobConf)" />
- <getworkoutputpath href="#getWorkOutputPath(org.apache.hadoop.mapred.JobConf)" />
- <setoutputpath href="#setOutputPath(org.apache.hadoop.mapred.JobConf,%20org.apache.hadoop.fs.Path)" />
- <setcompressoutput href="#setCompressOutput(org.apache.hadoop.mapred.JobConf,%20boolean)" />
- <setoutputcompressorclass href="#setOutputCompressorClass(org.apache.hadoop.mapred.JobConf,%20java.lang.Class)" />
- </fileoutputformat>
- <filesplit href="FileSplit.html" />
- <inputformat href="InputFormat.html" />
- <inputsplit href="InputSplit.html" />
- <isolationrunner href="IsolationRunner.html" />
- <jobclient href="JobClient.html">
- <runjob href="#runJob(org.apache.hadoop.mapred.JobConf)" />
- <submitjob href="#submitJob(org.apache.hadoop.mapred.JobConf)" />
- </jobclient>
- <jobconf href="JobConf.html">
- <setnummaptasks href="#setNumMapTasks(int)" />
- <setnumreducetasks href="#setNumReduceTasks(int)" />
- <setoutputkeycomparatorclass href="#setOutputKeyComparatorClass(java.lang.Class)" />
- <setoutputvaluegroupingcomparator href="#setOutputValueGroupingComparator(java.lang.Class)" />
- <setcombinerclass href="#setCombinerClass(java.lang.Class)" />
- <setmapdebugscript href="#setMapDebugScript(java.lang.String)" />
- <setreducedebugscript href="#setReduceDebugScript(java.lang.String)" />
- <setmapspeculativeexecution href="#setMapSpeculativeExecution(boolean)" />
- <setreducespeculativeexecution href="#setReduceSpeculativeExecution(boolean)" />
- <setmaxmapattempts href="#setMaxMapAttempts(int)" />
- <setmaxreduceattempts href="#setMaxReduceAttempts(int)" />
- <setmaxmaptaskfailurespercent href="#setMaxMapTaskFailuresPercent(int)" />
- <setmaxreducetaskfailurespercent href="#setMaxReduceTaskFailuresPercent(int)" />
- <setjobendnotificationuri href="#setJobEndNotificationURI(java.lang.String)" />
- <setcompressmapoutput href="#setCompressMapOutput(boolean)" />
- <setmapoutputcompressorclass href="#setMapOutputCompressorClass(java.lang.Class)" />
- <setprofileenabled href="#setProfileEnabled(boolean)" />
- <setprofiletaskrange href="#setProfileTaskRange(boolean,%20java.lang.String)" />
- <setprofileparams href="#setProfileParams(java.lang.String)" />
- <setnumtaskstoexecuteperjvm href="#setNumTasksToExecutePerJvm(int)" />
- <setqueuename href="#setQueueName(java.lang.String)" />
- <getjoblocaldir href="#getJobLocalDir()" />
- <getjar href="#getJar()" />
- </jobconf>
- <jobconfigurable href="JobConfigurable.html">
- <configure href="#configure(org.apache.hadoop.mapred.JobConf)" />
- </jobconfigurable>
- <jobcontrol href="jobcontrol/">
- <package-summary href="package-summary.html" />
- </jobcontrol>
- <mapper href="Mapper.html">
- <map href="#map(K1, V1, org.apache.hadoop.mapred.OutputCollector, org.apache.hadoop.mapred.Reporter)" />
- </mapper>
- <outputcollector href="OutputCollector.html">
- <collect href="#collect(K, V)" />
- </outputcollector>
- <outputcommitter href="OutputCommitter.html" />
- <outputformat href="OutputFormat.html" />
- <outputlogfilter href="OutputLogFilter.html" />
- <sequencefileoutputformat href="SequenceFileOutputFormat.html">
- <setoutputcompressiontype href="#setOutputCompressionType(org.apache.hadoop.mapred.JobConf,%20org.apache.hadoop.io.SequenceFile.CompressionType)" />
- </sequencefileoutputformat>
- <partitioner href="Partitioner.html" />
- <recordreader href="RecordReader.html" />
- <recordwriter href="RecordWriter.html" />
- <reducer href="Reducer.html">
- <reduce href="#reduce(K2, java.util.Iterator, org.apache.hadoop.mapred.OutputCollector, org.apache.hadoop.mapred.Reporter)" />
- </reducer>
- <reporter href="Reporter.html">
- <incrcounterEnum href="#incrCounter(java.lang.Enum, long)" />
- <incrcounterString href="#incrCounter(java.lang.String, java.lang.String, long amount)" />
- </reporter>
- <runningjob href="RunningJob.html" />
- <skipbadrecords href="SkipBadRecords.html">
- <setmappermaxskiprecords href="#setMapperMaxSkipRecords(org.apache.hadoop.conf.Configuration, long)"/>
- <setreducermaxskipgroups href="#setReducerMaxSkipGroups(org.apache.hadoop.conf.Configuration, long)"/>
- <setattemptsTostartskipping href="#setAttemptsToStartSkipping(org.apache.hadoop.conf.Configuration, int)"/>
- <setskipoutputpath href="#setSkipOutputPath(org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path)"/>
- <counter_map_processed_records href="#COUNTER_MAP_PROCESSED_RECORDS"/>
- <counter_reduce_processed_groups href="#COUNTER_REDUCE_PROCESSED_GROUPS"/>
- </skipbadrecords>
- <textinputformat href="TextInputFormat.html" />
- <textoutputformat href="TextOutputFormat.html" />
- <lib href="lib/">
- <package-summary href="package-summary.html" />
- <hashpartitioner href="HashPartitioner.html" />
- <keyfieldbasedpartitioner href="KeyFieldBasedPartitioner.html" />
- <keyfieldbasedcomparator href="KeyFieldBasedComparator.html" />
- <lazyoutputformat href="LazyOutputFormat.html" />
- <aggregate href="aggregate/">
- <package-summary href="package-summary.html" />
- </aggregate>
- </lib>
- <pipes href="pipes/">
- <package-summary href="package-summary.html" />
- </pipes>
- </mapred>
- <net href="net/">
- <dnstoswitchmapping href="DNSToSwitchMapping.html">
- <resolve href="#resolve(java.util.List)" />
- </dnstoswitchmapping>
- </net>
- <streaming href="streaming/">
- <package-summary href="package-summary.html" />
- </streaming>
- <util href="util/">
- <genericoptionsparser href="GenericOptionsParser.html" />
- <progress href="Progress.html" />
- <tool href="Tool.html" />
- <toolrunner href="ToolRunner.html">
- <run href="#run(org.apache.hadoop.util.Tool, java.lang.String[])" />
- </toolrunner>
- </util>
- </hadoop>
- </apache>
- </org>
- </api>
- </external-refs>
-
-</site>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/tabs.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/tabs.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/tabs.xml
deleted file mode 100644
index ffd556c..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/tabs.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!DOCTYPE tabs PUBLIC "-//APACHE//DTD Cocoon Documentation Tab V1.0//EN"
- "http://forrest.apache.org/dtd/tab-cocoon-v10.dtd">
-
-<tabs software="Hadoop"
- title="Hadoop"
- copyright="The Apache Software Foundation"
- xmlns:xlink="http://www.w3.org/1999/xlink">
-
- <!-- The rules are:
- @dir will always have /index.html added.
- @href is not modified unless it is root-relative and obviously specifies a
- directory (ends in '/'), in which case /index.html will be added
- -->
-
- <tab label="Project" href="http://hadoop.apache.org/common/" />
- <tab label="Wiki" href="http://wiki.apache.org/hadoop" />
- <tab label="Common 0.24 Documentation" dir="" />
-
-</tabs>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/architecture.gif
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/architecture.gif b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/architecture.gif
deleted file mode 100644
index 8d84a23..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/architecture.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/common-logo.jpg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/common-logo.jpg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/common-logo.jpg
deleted file mode 100644
index 8ba52cd..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/common-logo.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/core-logo.gif
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/core-logo.gif b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/core-logo.gif
deleted file mode 100644
index 57879bb..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/core-logo.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/favicon.ico
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/favicon.ico b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/favicon.ico
deleted file mode 100644
index 161bcf7..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/favicon.ico and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo-big.jpg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo-big.jpg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo-big.jpg
deleted file mode 100644
index 0c6996c..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo-big.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo.jpg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo.jpg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo.jpg
deleted file mode 100644
index 809525d..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hadoop-logo.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.gif
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.gif b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.gif
deleted file mode 100644
index 7bbe150..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.odg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.odg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.odg
deleted file mode 100644
index dae62bf..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.odg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.png
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.png b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.png
deleted file mode 100644
index 70e434c..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsarchitecture.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.gif
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.gif b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.gif
deleted file mode 100644
index b01f56f..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.odg
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.odg b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.odg
deleted file mode 100644
index e5d9227..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.odg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.png
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.png b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.png
deleted file mode 100644
index 40ea2ec..0000000
Binary files a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/resources/images/hdfsdatanodes.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/skinconf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/skinconf.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/skinconf.xml
deleted file mode 100644
index 3c93d7f..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/skinconf.xml
+++ /dev/null
@@ -1,366 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
-Skin configuration file. This file contains details of your project,
-which will be used to configure the chosen Forrest skin.
--->
-
-<!DOCTYPE skinconfig PUBLIC "-//APACHE//DTD Skin Configuration V0.6-3//EN" "http://forrest.apache.org/dtd/skinconfig-v06-3.dtd">
-<skinconfig>
- <!-- To enable lucene search add provider="lucene" (default is google).
- Add box-location="alt" to move the search box to an alternate location
- (if the skin supports it) and box-location="all" to show it in all
- available locations on the page. Remove the <search> element to show
- no search box. @domain will enable sitesearch for the specific domain with google.
- In other words google will search the @domain for the query string.
-
- -->
- <search name="Lucene" domain="hadoop.apache.org" provider="google"/>
-
- <!-- Disable the print link? If enabled, invalid HTML 4.0.1 -->
- <disable-print-link>true</disable-print-link>
- <!-- Disable the PDF link? -->
- <disable-pdf-link>false</disable-pdf-link>
- <!-- Disable the POD link? -->
- <disable-pod-link>true</disable-pod-link>
- <!-- Disable the Text link? FIXME: NOT YET IMPLEMENETED. -->
- <disable-txt-link>true</disable-txt-link>
- <!-- Disable the xml source link? -->
- <!-- The xml source link makes it possible to access the xml rendition
- of the source frim the html page, and to have it generated statically.
- This can be used to enable other sites and services to reuse the
- xml format for their uses. Keep this disabled if you don't want other
- sites to easily reuse your pages.-->
- <disable-xml-link>true</disable-xml-link>
-
- <!-- Disable navigation icons on all external links? -->
- <disable-external-link-image>true</disable-external-link-image>
-
- <!-- Disable w3c compliance links?
- Use e.g. align="center" to move the compliance links logos to
- an alternate location default is left.
- (if the skin supports it) -->
- <disable-compliance-links>true</disable-compliance-links>
-
- <!-- Render mailto: links unrecognisable by spam harvesters? -->
- <obfuscate-mail-links>false</obfuscate-mail-links>
-
- <!-- Disable the javascript facility to change the font size -->
- <disable-font-script>true</disable-font-script>
-
- <!-- project logo -->
- <project-name>Hadoop</project-name>
- <project-description>Scalable Computing Platform</project-description>
- <project-url>http://hadoop.apache.org/core/</project-url>
- <project-logo>images/common-logo.jpg</project-logo>
-
- <!-- group logo -->
- <group-name>Hadoop</group-name>
- <group-description>Apache Hadoop</group-description>
- <group-url>http://hadoop.apache.org/</group-url>
- <group-logo>images/hadoop-logo.jpg</group-logo>
-
- <!-- optional host logo (e.g. sourceforge logo)
- default skin: renders it at the bottom-left corner -->
- <host-url></host-url>
- <host-logo></host-logo>
-
- <!-- relative url of a favicon file, normally favicon.ico -->
- <favicon-url>images/favicon.ico</favicon-url>
-
- <!-- The following are used to construct a copyright statement -->
- <year>2009</year>
- <vendor>The Apache Software Foundation.</vendor>
- <copyright-link>http://www.apache.org/licenses/</copyright-link>
-
- <!-- Some skins use this to form a 'breadcrumb trail' of links.
- Use location="alt" to move the trail to an alternate location
- (if the skin supports it).
- Omit the location attribute to display the trail in the default location.
- Use location="none" to not display the trail (if the skin supports it).
- For some skins just set the attributes to blank.
- -->
- <trail>
- <link1 name="Apache" href="http://www.apache.org/"/>
- <link2 name="Hadoop" href="http://hadoop.apache.org/"/>
- <link3 name="Common" href="http://hadoop.apache.org/common/"/>
- </trail>
-
- <!-- Configure the TOC, i.e. the Table of Contents.
- @max-depth
- how many "section" levels need to be included in the
- generated Table of Contents (TOC).
- @min-sections
- Minimum required to create a TOC.
- @location ("page","menu","page,menu", "none")
- Where to show the TOC.
- -->
- <toc max-depth="2" min-sections="1" location="page"/>
-
- <!-- Heading types can be clean|underlined|boxed -->
- <headings type="clean"/>
-
- <!-- The optional feedback element will be used to construct a
- feedback link in the footer with the page pathname appended:
- <a href="@href">{@to}</a>
- <feedback to="webmaster@foo.com"
- href="mailto:webmaster@foo.com?subject=Feedback " >
- Send feedback about the website to:
- </feedback>
- -->
- <!--
- extra-css - here you can define custom css-elements that are
- a. overriding the fallback elements or
- b. adding the css definition from new elements that you may have
- used in your documentation.
- -->
- <extra-css>
- <!--Example of b.
- To define the css definition of a new element that you may have used
- in the class attribute of a <p> node.
- e.g. <p class="quote"/>
- -->
- p.quote {
- margin-left: 2em;
- padding: .5em;
- background-color: #f0f0f0;
- font-family: monospace;
- }
-
- <!--Headers -->
- #content h1 {
- margin-bottom: .5em;
- font-size: 185%; color: black;
- font-family: arial;
- }
- h2, .h3 { font-size: 175%; color: black; font-family: arial; }
- h3, .h4 { font-size: 135%; color: black; font-family: arial; margin-bottom: 0.5em; }
- h4, .h5 { font-size: 125%; color: black; font-style: italic; font-weight: bold; font-family: arial; }
- h5, h6 { font-size: 110%; color: #363636; font-weight: bold; }
-
- <!--Code Background -->
- pre.code {
- margin-left: 0em;
- padding: 0.5em;
- background-color: rgb(241,239,231);
- font-family: monospace;
- }
-
- </extra-css>
-
- <colors>
- <!-- These values are used for the generated CSS files. -->
-
- <!-- Krysalis -->
-<!--
- <color name="header" value="#FFFFFF"/>
-
- <color name="tab-selected" value="#a5b6c6" link="#000000" vlink="#000000" hlink="#000000"/>
- <color name="tab-unselected" value="#F7F7F7" link="#000000" vlink="#000000" hlink="#000000"/>
- <color name="subtab-selected" value="#a5b6c6" link="#000000" vlink="#000000" hlink="#000000"/>
- <color name="subtab-unselected" value="#a5b6c6" link="#000000" vlink="#000000" hlink="#000000"/>
-
- <color name="heading" value="#a5b6c6"/>
- <color name="subheading" value="#CFDCED"/>
-
- <color name="navstrip" value="#CFDCED" font="#000000" link="#000000" vlink="#000000" hlink="#000000"/>
- <color name="toolbox" value="#a5b6c6"/>
- <color name="border" value="#a5b6c6"/>
-
- <color name="menu" value="#F7F7F7" link="#000000" vlink="#000000" hlink="#000000"/>
- <color name="dialog" value="#F7F7F7"/>
-
- <color name="body" value="#ffffff" link="#0F3660" vlink="#009999" hlink="#000066"/>
-
- <color name="table" value="#a5b6c6"/>
- <color name="table-cell" value="#ffffff"/>
- <color name="highlight" value="#ffff00"/>
- <color name="fixme" value="#cc6600"/>
- <color name="note" value="#006699"/>
- <color name="warning" value="#990000"/>
- <color name="code" value="#a5b6c6"/>
-
- <color name="footer" value="#a5b6c6"/>
--->
-
- <!-- Forrest -->
-<!--
- <color name="header" value="#294563"/>
-
- <color name="tab-selected" value="#4a6d8c" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
- <color name="tab-unselected" value="#b5c7e7" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
- <color name="subtab-selected" value="#4a6d8c" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
- <color name="subtab-unselected" value="#4a6d8c" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
-
- <color name="heading" value="#294563"/>
- <color name="subheading" value="#4a6d8c"/>
-
- <color name="navstrip" value="#cedfef" font="#0F3660" link="#0F3660" vlink="#0F3660" hlink="#000066"/>
- <color name="toolbox" value="#4a6d8c"/>
- <color name="border" value="#294563"/>
-
- <color name="menu" value="#4a6d8c" font="#cedfef" link="#ffffff" vlink="#ffffff" hlink="#ffcf00"/>
- <color name="dialog" value="#4a6d8c"/>
-
- <color name="body" value="#ffffff" link="#0F3660" vlink="#009999" hlink="#000066"/>
-
- <color name="table" value="#7099C5"/>
- <color name="table-cell" value="#f0f0ff"/>
- <color name="highlight" value="#ffff00"/>
- <color name="fixme" value="#cc6600"/>
- <color name="note" value="#006699"/>
- <color name="warning" value="#990000"/>
- <color name="code" value="#CFDCED"/>
-
- <color name="footer" value="#cedfef"/>
--->
-
- <!-- Collabnet -->
-<!--
- <color name="header" value="#003366"/>
-
- <color name="tab-selected" value="#dddddd" link="#555555" vlink="#555555" hlink="#555555"/>
- <color name="tab-unselected" value="#999999" link="#ffffff" vlink="#ffffff" hlink="#ffffff"/>
- <color name="subtab-selected" value="#cccccc" link="#000000" vlink="#000000" hlink="#000000"/>
- <color name="subtab-unselected" value="#cccccc" link="#555555" vlink="#555555" hlink="#555555"/>
-
- <color name="heading" value="#003366"/>
- <color name="subheading" value="#888888"/>
-
- <color name="navstrip" value="#dddddd" font="#555555"/>
- <color name="toolbox" value="#dddddd" font="#555555"/>
- <color name="border" value="#999999"/>
-
- <color name="menu" value="#ffffff"/>
- <color name="dialog" value="#eeeeee"/>
-
- <color name="body" value="#ffffff"/>
-
- <color name="table" value="#ccc"/>
- <color name="table-cell" value="#ffffff"/>
- <color name="highlight" value="#ffff00"/>
- <color name="fixme" value="#cc6600"/>
- <color name="note" value="#006699"/>
- <color name="warning" value="#990000"/>
- <color name="code" value="#003366"/>
-
- <color name="footer" value="#ffffff"/>
--->
- <!-- Lenya using pelt-->
-<!--
- <color name="header" value="#ffffff"/>
-
- <color name="tab-selected" value="#4C6C8F" link="#ffffff" vlink="#ffffff" hlink="#ffffff"/>
- <color name="tab-unselected" value="#E5E4D9" link="#000000" vlink="#000000" hlink="#000000"/>
- <color name="subtab-selected" value="#000000" link="#000000" vlink="#000000" hlink="#000000"/>
- <color name="subtab-unselected" value="#E5E4D9" link="#000000" vlink="#000000" hlink="#000000"/>
-
- <color name="heading" value="#E5E4D9"/>
- <color name="subheading" value="#000000"/>
- <color name="published" value="#4C6C8F" font="#FFFFFF"/>
- <color name="feedback" value="#4C6C8F" font="#FFFFFF" align="center"/>
- <color name="navstrip" value="#E5E4D9" font="#000000"/>
-
- <color name="toolbox" value="#CFDCED" font="#000000"/>
-
- <color name="border" value="#999999"/>
- <color name="menu" value="#4C6C8F" font="#ffffff" link="#ffffff" vlink="#ffffff" hlink="#ffffff" current="#FFCC33" />
- <color name="menuheading" value="#cfdced" font="#000000" />
- <color name="searchbox" value="#E5E4D9" font="#000000"/>
-
- <color name="dialog" value="#CFDCED"/>
- <color name="body" value="#ffffff" />
-
- <color name="table" value="#ccc"/>
- <color name="table-cell" value="#ffffff"/>
- <color name="highlight" value="#ffff00"/>
- <color name="fixme" value="#cc6600"/>
- <color name="note" value="#006699"/>
- <color name="warning" value="#990000"/>
- <color name="code" value="#003366"/>
-
- <color name="footer" value="#E5E4D9"/>
--->
- </colors>
-
- <!-- Settings specific to PDF output. -->
- <pdf>
- <!--
- Supported page sizes are a0, a1, a2, a3, a4, a5, executive,
- folio, legal, ledger, letter, quarto, tabloid (default letter).
- Supported page orientations are portrait, landscape (default
- portrait).
- Supported text alignments are left, right, justify (default left).
- -->
- <page size="letter" orientation="portrait" text-align="left"/>
-
- <!--
- Margins can be specified for top, bottom, inner, and outer
- edges. If double-sided="false", the inner edge is always left
- and the outer is always right. If double-sided="true", the
- inner edge will be left on odd pages, right on even pages,
- the outer edge vice versa.
- Specified below are the default settings.
- -->
- <margins double-sided="false">
- <top>1in</top>
- <bottom>1in</bottom>
- <inner>1.25in</inner>
- <outer>1in</outer>
- </margins>
-
- <!--
- Print the URL text next to all links going outside the file
- -->
- <show-external-urls>false</show-external-urls>
-
- <!--
- Disable the copyright footer on each page of the PDF.
- A footer is composed for each page. By default, a "credit" with role=pdf
- will be used, as explained below. Otherwise a copyright statement
- will be generated. This latter can be disabled.
- -->
- <disable-copyright-footer>false</disable-copyright-footer>
- </pdf>
-
- <!-- Credits are typically rendered as a set of small clickable
- images in the page footer.
- Use box-location="alt" to move the credit to an alternate location
- (if the skin supports it).
- -->
- <credits>
- <credit box-location="alt">
- <name>Built with Apache Forrest</name>
- <url>http://forrest.apache.org/</url>
- <image>images/built-with-forrest-button.png</image>
- <width>88</width>
- <height>31</height>
- </credit>
- <!-- A credit with @role="pdf" will be used to compose a footer
- for each page in the PDF, using either "name" or "url" or both.
- -->
- <!--
- <credit role="pdf">
- <name>Built with Apache Forrest</name>
- <url>http://forrest.apache.org/</url>
- </credit>
- -->
- </credits>
-
-</skinconfig>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-common-project/hadoop-common/src/main/docs/status.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/status.xml b/hadoop-common-project/hadoop-common/src/main/docs/status.xml
deleted file mode 100644
index a820726..0000000
--- a/hadoop-common-project/hadoop-common/src/main/docs/status.xml
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<status>
-
- <developers>
- <person name="Joe Bloggs" email="joe@joescompany.org" id="JB" />
- <!-- Add more people here -->
- </developers>
-
- <changes>
- <!-- Add new releases here -->
- <release version="0.1" date="unreleased">
- <!-- Some action types have associated images. By default, images are
- defined for 'add', 'fix', 'remove', 'update' and 'hack'. If you add
- src/documentation/resources/images/<foo>.jpg images, these will
- automatically be used for entries of type <foo>. -->
-
- <action dev="JB" type="add" context="admin">
- Initial Import
- </action>
- <!-- Sample action:
- <action dev="JB" type="fix" due-to="Joe Contributor"
- due-to-email="joec@apache.org" fixes-bug="123">
- Fixed a bug in the Foo class.
- </action>
- -->
- </release>
- </changes>
-
- <todo>
- <actions priority="high">
- <action context="docs" dev="JB">
- Customize this template project with your project's details. This
- TODO list is generated from 'status.xml'.
- </action>
- <action context="docs" dev="JB">
- Add lots of content. XML content goes in
- <code>src/documentation/content/xdocs</code>, or wherever the
- <code>${project.xdocs-dir}</code> property (set in
- <code>forrest.properties</code>) points.
- </action>
- <action context="feedback" dev="JB">
- Mail <link
- href="mailto:forrest-dev@xml.apache.org">forrest-dev@xml.apache.org</link>
- with feedback.
- </action>
- </actions>
- <!-- Add todo items. @context is an arbitrary string. Eg:
- <actions priority="high">
- <action context="code" dev="SN">
- </action>
- </actions>
- <actions priority="medium">
- <action context="docs" dev="open">
- </action>
- </actions>
- -->
- </todo>
-
-</status>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 34338d1..3eeadb6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -373,7 +373,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<exclude>.gitattributes</exclude>
<exclude>.idea/**</exclude>
<exclude>src/main/conf/*</exclude>
- <exclude>src/main/docs/**</exclude>
<exclude>dev-support/findbugsExcludeFile.xml</exclude>
<exclude>dev-support/checkstyle*</exclude>
<exclude>dev-support/jdiff/**</exclude>
@@ -390,7 +389,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<exclude>src/main/native/util/tree.h</exclude>
<exclude>src/test/aop/org/apache/hadoop/hdfs/server/datanode/DataXceiverAspects.aj</exclude>
<exclude>src/main/webapps/datanode/robots.txt</exclude>
- <exclude>src/main/docs/releasenotes.html</exclude>
<exclude>src/contrib/**</exclude>
<exclude>src/site/resources/images/*</exclude>
<exclude>src/main/webapps/static/bootstrap-3.0.2/**</exclude>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b17498d6/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index c26aa9a..e712910 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -246,28 +246,12 @@
<mkdir dir="${project.build.directory}/docs-src"/>
- <copy todir="${project.build.directory}/docs-src">
- <fileset dir="${basedir}/src/main/docs"/>
- </copy>
-
<!-- Docs -->
- <copy file="${project.build.directory}/docs-src/releasenotes.html"
- todir="${project.build.directory}/site"/>
<style basedir="${basedir}/src/main/resources"
destdir="${project.build.directory}/site"
includes="core-default.xml"
style="${basedir}/src/main/xsl/configuration.xsl"/>
- <!-- Convert 'CHANGES.txt' to 'changes.html" -->
- <exec executable="perl" input="${basedir}/../CHANGES.txt"
- output="${project.build.directory}/site/changes.html"
- failonerror="true">
- <arg value="${project.build.directory}/docs-src/changes/changes2html.pl"/>
- </exec>
- <copy todir="${project.build.directory}/site">
- <fileset dir="${project.build.directory}/docs-src/changes" includes="*.css"/>
- </copy>
-
<!-- Jdiff -->
<mkdir dir="${project.build.directory}/site/jdiff/xml"/>
[17/50] [abbrv] hadoop git commit: HADOOP-11983.
HADOOP_USER_CLASSPATH_FIRST works the opposite of what it is supposed to do
(Sangjin Lee via aw)
Posted by zj...@apache.org.
HADOOP-11983. HADOOP_USER_CLASSPATH_FIRST works the opposite of what it is supposed to do (Sangjin Lee via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b914ccaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b914ccaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b914ccaf
Branch: refs/heads/YARN-2928
Commit: b914ccafc9647f47e1e667fc10d051d9f7f31e1e
Parents: f47c0e9
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 11:29:58 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/main/bin/hadoop-functions.sh | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b914ccaf/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 71b3388..69f5aa7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -474,6 +474,9 @@ Trunk (Unreleased)
HADOOP-11406. xargs -P is not portable (Kengo Seki via aw)
+ HADOOP-11983. HADOOP_USER_CLASSPATH_FIRST works the opposite of what it is
+ supposed to do (Sangjin Lee via aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b914ccaf/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
old mode 100644
new mode 100755
index 5556f2f..be65d7f
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -761,11 +761,11 @@ function hadoop_add_to_classpath_userpath
if [[ -z "${HADOOP_USE_CLIENT_CLASSLOADER}" ]]; then
if [[ -z "${HADOOP_USER_CLASSPATH_FIRST}" ]]; then
for ((i=j; i>=0; i--)); do
- hadoop_add_classpath "${array[$i]}" before
+ hadoop_add_classpath "${array[$i]}" after
done
else
for ((i=0; i<=j; i++)); do
- hadoop_add_classpath "${array[$i]}" after
+ hadoop_add_classpath "${array[$i]}" before
done
fi
fi
[24/50] [abbrv] hadoop git commit: HADOOP-11959. WASB should
configure client side socket timeout in storage client blob request options.
Contributed by Ivan Mitic.
Posted by zj...@apache.org.
HADOOP-11959. WASB should configure client side socket timeout in storage client blob request options. Contributed by Ivan Mitic.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44f4df57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44f4df57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44f4df57
Branch: refs/heads/YARN-2928
Commit: 44f4df57bfe7a078894cc4c05b07c0e3a292412f
Parents: 4b4aeba
Author: cnauroth <cn...@apache.org>
Authored: Thu May 28 12:31:06 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:56 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-project/pom.xml | 2 +-
.../hadoop/fs/azure/AzureNativeFileSystemStore.java | 13 ++-----------
.../org/apache/hadoop/fs/azure/StorageInterface.java | 6 +++---
.../apache/hadoop/fs/azure/StorageInterfaceImpl.java | 4 ++--
.../apache/hadoop/fs/azure/MockStorageInterface.java | 4 ++--
.../fs/azure/TestAzureFileSystemErrorConditions.java | 1 +
.../apache/hadoop/fs/azure/TestBlobDataValidation.java | 1 +
8 files changed, 15 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6aa224c..aba9087 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -791,6 +791,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11930. test-patch in offline mode should tell maven to be in
offline mode (Sean Busbey via aw)
+ HADOOP-11959. WASB should configure client side socket timeout in storage
+ client blob request options. (Ivan Mitic via cnauroth)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 94e1f5e..a5d4ed8 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -942,7 +942,7 @@
<dependency>
<groupId>com.microsoft.azure</groupId>
<artifactId>azure-storage</artifactId>
- <version>2.0.0</version>
+ <version>2.2.0</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 3267d8b..69bda06 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -2434,15 +2434,6 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
//
CloudBlobWrapper dstBlob = getBlobReference(dstKey);
- // TODO: Remove at the time when we move to Azure Java SDK 1.2+.
- // This is the workaround provided by Azure Java SDK team to
- // mitigate the issue with un-encoded x-ms-copy-source HTTP
- // request header. Azure sdk version before 1.2+ does not encode this
- // header what causes all URIs that have special (category "other")
- // characters in the URI not to work with startCopyFromBlob when
- // specified as source (requests fail with HTTP 403).
- URI srcUri = new URI(srcBlob.getUri().toASCIIString());
-
// Rename the source blob to the destination blob by copying it to
// the destination blob then deleting it.
//
@@ -2451,7 +2442,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// a more intensive exponential retry policy when the cluster is getting
// throttled.
try {
- dstBlob.startCopyFromBlob(srcUri, null, getInstrumentedContext());
+ dstBlob.startCopyFromBlob(srcBlob, null, getInstrumentedContext());
} catch (StorageException se) {
if (se.getErrorCode().equals(
StorageErrorCode.SERVER_BUSY.toString())) {
@@ -2475,7 +2466,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
options.setRetryPolicyFactory(new RetryExponentialRetry(
copyBlobMinBackoff, copyBlobDeltaBackoff, copyBlobMaxBackoff,
copyBlobMaxRetries));
- dstBlob.startCopyFromBlob(srcUri, options, getInstrumentedContext());
+ dstBlob.startCopyFromBlob(srcBlob, options, getInstrumentedContext());
} else {
throw se;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
index e89151d..ce5f749 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterface.java
@@ -381,8 +381,8 @@ abstract class StorageInterface {
* Copies an existing blob's contents, properties, and metadata to this instance of the <code>CloudBlob</code>
* class, using the specified operation context.
*
- * @param source
- * A <code>java.net.URI</code> The URI of a source blob.
+ * @param sourceBlob
+ * A <code>CloudBlob</code> object that represents the source blob to copy.
* @param options
* A {@link BlobRequestOptions} object that specifies any additional options for the request. Specifying
* <code>null</code> will use the default request options from the associated service client (
@@ -397,7 +397,7 @@ abstract class StorageInterface {
* @throws URISyntaxException
*
*/
- public abstract void startCopyFromBlob(URI source,
+ public abstract void startCopyFromBlob(CloudBlobWrapper sourceBlob,
BlobRequestOptions options, OperationContext opContext)
throws StorageException, URISyntaxException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
index 90d4d88..382ff66 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/StorageInterfaceImpl.java
@@ -393,10 +393,10 @@ class StorageInterfaceImpl extends StorageInterface {
}
@Override
- public void startCopyFromBlob(URI source, BlobRequestOptions options,
+ public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
OperationContext opContext)
throws StorageException, URISyntaxException {
- getBlob().startCopyFromBlob(source,
+ getBlob().startCopyFromBlob(((CloudBlobWrapperImpl)sourceBlob).blob,
null, null, options, opContext);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
index cde0e38..9f84f4b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
@@ -429,9 +429,9 @@ public class MockStorageInterface extends StorageInterface {
}
@Override
- public void startCopyFromBlob(URI source, BlobRequestOptions options,
+ public void startCopyFromBlob(CloudBlobWrapper sourceBlob, BlobRequestOptions options,
OperationContext opContext) throws StorageException, URISyntaxException {
- backingStore.copy(convertUriToDecodedString(source), convertUriToDecodedString(uri));
+ backingStore.copy(convertUriToDecodedString(sourceBlob.getUri()), convertUriToDecodedString(uri));
//TODO: set the backingStore.properties.CopyState and
// update azureNativeFileSystemStore.waitForCopyToComplete
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
index ace57dc..810bcf7 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java
@@ -205,6 +205,7 @@ public class TestAzureFileSystemErrorConditions {
@Override
public boolean isTargetConnection(HttpURLConnection connection) {
return connection.getRequestMethod().equals("PUT")
+ && connection.getURL().getQuery() != null
&& connection.getURL().getQuery().contains("blocklist");
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44f4df57/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
index 9237ade..c40b7b5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobDataValidation.java
@@ -191,6 +191,7 @@ public class TestBlobDataValidation {
private static boolean isPutBlock(HttpURLConnection connection) {
return connection.getRequestMethod().equals("PUT")
+ && connection.getURL().getQuery() != null
&& connection.getURL().getQuery().contains("blockid");
}
[14/50] [abbrv] hadoop git commit: HADOOP-12035. shellcheck plugin
displays a wrong version potentially (Kengo Seki via aw)
Posted by zj...@apache.org.
HADOOP-12035. shellcheck plugin displays a wrong version potentially (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3eb698cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3eb698cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3eb698cb
Branch: refs/heads/YARN-2928
Commit: 3eb698cb45863fd6e9899fc76983a3832a032ec3
Parents: 7dc7948
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 09:11:08 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:54 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/shellcheck.sh | 2 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3eb698cb/dev-support/test-patch.d/shellcheck.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
index 5f38b6a..9277ea5 100755
--- a/dev-support/test-patch.d/shellcheck.sh
+++ b/dev-support/test-patch.d/shellcheck.sh
@@ -72,7 +72,7 @@ function shellcheck_preapply
start_clock
# shellcheck disable=SC2016
- SHELLCHECK_VERSION=$(shellcheck --version | ${GREP} version: | ${AWK} '{print $NF}')
+ SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
echo "Running shellcheck against all identifiable shell scripts"
pushd "${BASEDIR}" >/dev/null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3eb698cb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 48e9f7d..1009d1a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -765,6 +765,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12004. test-patch breaks with reexec in certain situations (Sean
Busbey via aw)
+ HADOOP-12035. shellcheck plugin displays a wrong version potentially
+ (Kengo Seki via aw)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
[35/50] [abbrv] hadoop git commit: HADOOP-11885. hadoop-dist
dist-layout-stitching.sh does not work with dash. (wang)
Posted by zj...@apache.org.
HADOOP-11885. hadoop-dist dist-layout-stitching.sh does not work with dash. (wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/547b9dd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/547b9dd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/547b9dd5
Branch: refs/heads/YARN-2928
Commit: 547b9dd5a3aabe50818b1372e1cf6a9742478068
Parents: 58b91c9
Author: Andrew Wang <an...@cloudera.com>
Authored: Fri May 29 14:54:18 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:58 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-common/pom.xml | 2 +-
hadoop-common-project/hadoop-kms/pom.xml | 4 ++--
hadoop-dist/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 2 +-
.../hadoop-mapreduce-client-nativetask/pom.xml | 2 +-
hadoop-mapreduce-project/pom.xml | 4 ++--
hadoop-project-dist/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 2 +-
hadoop-yarn-project/pom.xml | 2 +-
pom.xml | 2 ++
12 files changed, 20 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index bc74696..e5d2e39 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -643,6 +643,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11242. Record the time of calling in tracing span of
IPC server. (Masatake Iwasaki via aajisaka)
+ HADOOP-11885. hadoop-dist dist-layout-stitching.sh does not work with dash.
+ (wang)
+
BUG FIXES
HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 3043889..c3a306b 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -666,7 +666,7 @@
<goals><goal>run</goal></goals>
<configuration>
<target>
- <exec executable="sh" failonerror="true" dir="${project.build.directory}/native">
+ <exec executable="${shell-executable}" failonerror="true" dir="${project.build.directory}/native">
<arg value="-c"/>
<arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_bulk_crc32"/>
<env key="SKIPTESTS" value="${skipTests}"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index c33b33e..ad2062b 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -383,7 +383,7 @@
cd "${project.build.directory}/tomcat.exp"
gzip -cd ../../downloads/apache-tomcat-${tomcat.version}.tar.gz | tar xf -
</echo>
- <exec executable="sh" dir="${project.build.directory}"
+ <exec executable="${shell-executable}" dir="${project.build.directory}"
failonerror="true">
<arg line="./tomcat-untar.sh"/>
</exec>
@@ -426,7 +426,7 @@
cd "${project.build.directory}"
tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
</echo>
- <exec executable="sh" dir="${project.build.directory}"
+ <exec executable="${shell-executable}" dir="${project.build.directory}"
failonerror="true">
<arg line="./dist-maketar.sh"/>
</exec>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index f894c01..f1644a8 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -191,7 +191,7 @@
echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}"
echo
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./dist-layout-stitching.sh"/>
</exec>
</target>
@@ -224,7 +224,7 @@
echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz"
echo
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./dist-tar-stitching.sh"/>
</exec>
</target>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 520e30f..4457597 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -517,7 +517,7 @@
cd "${project.build.directory}/tomcat.exp"
gzip -cd ../../downloads/apache-tomcat-${tomcat.version}.tar.gz | tar xf -
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./tomcat-untar.sh"/>
</exec>
@@ -557,7 +557,7 @@
cd "${project.build.directory}"
tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./dist-maketar.sh"/>
</exec>
</target>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 3eeadb6..f90644c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -674,7 +674,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<phase>test-compile</phase>
<configuration>
<target>
- <exec executable="sh">
+ <exec executable="${shell-executable}">
<arg value="-c"/>
<arg value="for i in {1..${testsThreadCount}}; do mkdir -p ${test.build.data}/$i; mkdir -p ${hadoop.tmp.dir}/$i; done"/>
</exec>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 3b0ee48..a5532f1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -197,7 +197,7 @@
<goals><goal>run</goal></goals>
<configuration>
<target>
- <exec executable="sh" failonerror="true" dir="${project.build.directory}/native/test">
+ <exec executable="${shell-executable}" failonerror="true" dir="${project.build.directory}/native/test">
<arg value="-c"/>
<arg value="[ x$SKIPTESTS = xtrue ] || sh test.sh"/>
<env key="SKIPTESTS" value="${skipTests}"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-mapreduce-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 088a3ba..0cbd928 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -170,7 +170,7 @@
cd "${project.build.directory}"
tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./dist-maketar.sh"/>
</exec>
</target>
@@ -254,7 +254,7 @@
cp -R $${LIB_DIR}/lib* $${TARGET_DIR}
fi
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./dist-copynativelibs.sh"/>
</exec>
</target>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index e712910..8df266f 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -361,7 +361,7 @@
fi
fi
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./dist-copynativelibs.sh"/>
</exec>
</target>
@@ -380,7 +380,7 @@
cd "${project.build.directory}"
tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./dist-maketar.sh"/>
</exec>
</target>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 8e135de..babed8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -203,7 +203,7 @@
<phase>test</phase>
<configuration>
<target>
- <exec executable="sh" failonerror="true" dir="${project.build.directory}/native">
+ <exec executable="${shell-executable}" failonerror="true" dir="${project.build.directory}/native">
<arg value="-c"/>
<arg value="[ x$SKIPTESTS = xtrue ] || test-container-executor"/>
<env key="SKIPTESTS" value="${skipTests}"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 736508b..e21e0b8 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -96,7 +96,7 @@
cd "${project.build.directory}"
tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz
</echo>
- <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./dist-maketar.sh"/>
</exec>
</target>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547b9dd5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8394324f..4e6f7a1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -105,6 +105,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
<maven-checkstyle-plugin.version>2.15</maven-checkstyle-plugin.version>
<checkstyle.version>6.6</checkstyle.version>
+
+ <shell-executable>bash</shell-executable>
</properties>
<modules>
[41/50] [abbrv] hadoop git commit: YARN-3740. Fixed the typo in the
configuration name: APPLICATION_HISTORY_PREFIX_MAX_APPS. Contributed by Xuan
Gong.
Posted by zj...@apache.org.
YARN-3740. Fixed the typo in the configuration name: APPLICATION_HISTORY_PREFIX_MAX_APPS. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0c491be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0c491be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0c491be
Branch: refs/heads/YARN-2928
Commit: d0c491bee6c5bc72f3cb06c01c8142349d666eb4
Parents: 312030a
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri May 29 18:16:48 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:59 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++--
.../ApplicationHistoryManagerOnTimelineStore.java | 4 ++--
.../TestApplicationHistoryClientService.java | 2 +-
4 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0c491be/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 906d9d4..339203c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -558,6 +558,9 @@ Release 2.8.0 - UNRELEASED
YARN-3716. Node-label-expression should be included by
ResourceRequestPBImpl.toString. (Xianyin Xin via wangda)
+ YARN-3740. Fixed the typo in the configuration name:
+ APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0c491be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 1d42c60..ba07c80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1500,9 +1500,9 @@ public class YarnConfiguration extends Configuration {
* REST API or application history protocol and shown in timeline
* server web ui.
*/
- public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+ public static final String APPLICATION_HISTORY_MAX_APPS =
APPLICATION_HISTORY_PREFIX + "max-applications";
- public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+ public static final long DEFAULT_APPLICATION_HISTORY_MAX_APPS = 10000;
/** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0c491be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index d3e7b5c..9bfd2d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -94,8 +94,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
maxLoadedApplications =
- conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
- YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
+ conf.getLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_MAX_APPS);
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0c491be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index ad03718..764ecb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -108,7 +108,7 @@ public class TestApplicationHistoryClientService {
// Create a historyManager, and set the max_apps can be loaded
// as 1.
Configuration conf = new YarnConfiguration();
- conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+ conf.setLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS, 1);
ApplicationHistoryManagerOnTimelineStore historyManager2 =
new ApplicationHistoryManagerOnTimelineStore(dataManager,
new ApplicationACLsManager(conf));
[36/50] [abbrv] hadoop git commit: YARN-3547. FairScheduler: Apps
that have no resource demand should not participate scheduling. (Xianyin Xin
via kasha)
Posted by zj...@apache.org.
YARN-3547. FairScheduler: Apps that have no resource demand should not participate scheduling. (Xianyin Xin via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b126e9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b126e9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b126e9b
Branch: refs/heads/YARN-2928
Commit: 6b126e9bba94c4a8180168e2803d754b8eb81a59
Parents: 547b9dd
Author: Karthik Kambatla <ka...@apache.org>
Authored: Fri May 29 15:17:02 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:58 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../scheduler/fair/FSLeafQueue.java | 43 ++++++++++----------
2 files changed, 24 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b126e9b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2fe1b02..4ddf11b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -379,6 +379,9 @@ Release 2.8.0 - UNRELEASED
YARN-3006. Improve the error message when attempting manual failover with
auto-failover enabled. (Akira AJISAKA via wangda)
+ YARN-3547. FairScheduler: Apps that have no resource demand should not participate
+ scheduling. (Xianyin Xin via kasha)
+
BUG FIXES
YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b126e9b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 04dbd2f..6779a1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.TreeSet;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
@@ -314,35 +315,33 @@ public class FSLeafQueue extends FSQueue {
return assigned;
}
- Comparator<Schedulable> comparator = policy.getComparator();
- writeLock.lock();
- try {
- Collections.sort(runnableApps, comparator);
- } finally {
- writeLock.unlock();
- }
- // Release write lock here for better performance and avoiding deadlocks.
- // runnableApps can be in unsorted state because of this section,
- // but we can accept it in practice since the probability is low.
+ // Apps that have resource demands.
+ TreeSet<FSAppAttempt> pendingForResourceApps =
+ new TreeSet<FSAppAttempt>(policy.getComparator());
readLock.lock();
try {
- for (FSAppAttempt sched : runnableApps) {
- if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) {
- continue;
- }
-
- assigned = sched.assignContainer(node);
- if (!assigned.equals(Resources.none())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Assigned container in queue:" + getName() + " " +
- "container:" + assigned);
- }
- break;
+ for (FSAppAttempt app : runnableApps) {
+ Resource pending = app.getAppAttemptResourceUsage().getPending();
+ if (!pending.equals(Resources.none())) {
+ pendingForResourceApps.add(app);
}
}
} finally {
readLock.unlock();
}
+ for (FSAppAttempt sched : pendingForResourceApps) {
+ if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) {
+ continue;
+ }
+ assigned = sched.assignContainer(node);
+ if (!assigned.equals(Resources.none())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Assigned container in queue:" + getName() + " " +
+ "container:" + assigned);
+ }
+ break;
+ }
+ }
return assigned;
}
[20/50] [abbrv] hadoop git commit: HADOOP-7947. Validate XMLs if a
relevant tool is available, when using scripts (Kengo Seki via aw)
Posted by zj...@apache.org.
HADOOP-7947. Validate XMLs if a relevant tool is available, when using scripts (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f47c0e9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f47c0e9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f47c0e9a
Branch: refs/heads/YARN-2928
Commit: f47c0e9ad030e8a15de32cec00a15e253ff52784
Parents: 82829c8
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 28 11:01:00 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:55 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop-common/src/main/bin/hadoop | 4 +
.../java/org/apache/hadoop/util/ConfTest.java | 368 +++++++++++++++++++
.../org/apache/hadoop/util/TestConfTest.java | 204 ++++++++++
4 files changed, 579 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f47c0e9a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8de5ae4..71b3388 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -48,6 +48,9 @@ Trunk (Unreleased)
HADOOP-9642. Configuration to resolve environment variables via
${env.VARIABLE} references (Kengo Seki via aw)
+ HADOOP-7947. Validate XMLs if a relevant tool is available, when using
+ scripts (Kengo Seki via aw)
+
IMPROVEMENTS
HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f47c0e9a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 64c3c13..9e7b3ef 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -27,6 +27,7 @@ function hadoop_usage()
echo " libraries availability"
echo " classpath prints the class path needed to get the"
echo " Hadoop jar and the required libraries"
+ echo " conftest validate configuration XML files"
echo " credential interact with credential providers"
echo " daemonlog get/set the log level for each daemon"
echo " distch path:owner:group:permisson"
@@ -127,6 +128,9 @@ case ${COMMAND} in
classpath)
hadoop_do_classpath_subcommand CLASS "$@"
;;
+ conftest)
+ CLASS=org.apache.hadoop.util.ConfTest
+ ;;
credential)
CLASS=org.apache.hadoop.security.alias.CredentialShell
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f47c0e9a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java
new file mode 100644
index 0000000..3f37f5a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+import java.util.Map.Entry;
+
+import javax.xml.namespace.QName;
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.Attribute;
+import javax.xml.stream.events.Characters;
+import javax.xml.stream.events.StartElement;
+import javax.xml.stream.events.XMLEvent;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * This class validates configuration XML files in ${HADOOP_CONF_DIR} or
+ * specified ones.
+ */
+@InterfaceAudience.Private
+public final class ConfTest {
+
+ private static final String USAGE =
+ "Usage: hadoop conftest [-conffile <path>|-h|--help]\n"
+ + " Options:\n"
+ + " \n"
+ + " -conffile <path>\n"
+ + " If not specified, the files in ${HADOOP_CONF_DIR}\n"
+ + " whose name end with .xml will be verified.\n"
+ + " If specified, that path will be verified.\n"
+ + " You can specify either a file or directory, and\n"
+ + " if a directory specified, the files in that directory\n"
+ + " whose name end with .xml will be verified.\n"
+ + " You can specify this option multiple times.\n"
+ + " -h, --help Print this help";
+
+ private static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR";
+
+ protected ConfTest() {
+ super();
+ }
+
+ private static List<NodeInfo> parseConf(InputStream in)
+ throws XMLStreamException {
+ QName configuration = new QName("configuration");
+ QName property = new QName("property");
+
+ List<NodeInfo> nodes = new ArrayList<NodeInfo>();
+ Stack<NodeInfo> parsed = new Stack<NodeInfo>();
+
+ XMLInputFactory factory = XMLInputFactory.newInstance();
+ XMLEventReader reader = factory.createXMLEventReader(in);
+
+ while (reader.hasNext()) {
+ XMLEvent event = reader.nextEvent();
+ if (event.isStartElement()) {
+ StartElement currentElement = event.asStartElement();
+ NodeInfo currentNode = new NodeInfo(currentElement);
+ if (parsed.isEmpty()) {
+ if (!currentElement.getName().equals(configuration)) {
+ return null;
+ }
+ } else {
+ NodeInfo parentNode = parsed.peek();
+ QName parentName = parentNode.getStartElement().getName();
+ if (parentName.equals(configuration)
+ && currentNode.getStartElement().getName().equals(property)) {
+ @SuppressWarnings("unchecked")
+ Iterator<Attribute> it = currentElement.getAttributes();
+ while (it.hasNext()) {
+ currentNode.addAttribute(it.next());
+ }
+ } else if (parentName.equals(property)) {
+ parentNode.addElement(currentElement);
+ }
+ }
+ parsed.push(currentNode);
+ } else if (event.isEndElement()) {
+ NodeInfo node = parsed.pop();
+ if (parsed.size() == 1) {
+ nodes.add(node);
+ }
+ } else if (event.isCharacters()) {
+ if (2 < parsed.size()) {
+ NodeInfo parentNode = parsed.pop();
+ StartElement parentElement = parentNode.getStartElement();
+ NodeInfo grandparentNode = parsed.peek();
+ if (grandparentNode.getElement(parentElement) == null) {
+ grandparentNode.setElement(parentElement, event.asCharacters());
+ }
+ parsed.push(parentNode);
+ }
+ }
+ }
+
+ return nodes;
+ }
+
+ public static List<String> checkConf(InputStream in) {
+ List<NodeInfo> nodes = null;
+ List<String> errors = new ArrayList<String>();
+
+ try {
+ nodes = parseConf(in);
+ if (nodes == null) {
+ errors.add("bad conf file: top-level element not <configuration>");
+ }
+ } catch (XMLStreamException e) {
+ errors.add("bad conf file: " + e.getMessage());
+ }
+
+ if (!errors.isEmpty()) {
+ return errors;
+ }
+
+ Map<String, List<Integer>> duplicatedProperties =
+ new HashMap<String, List<Integer>>();
+
+ for (NodeInfo node : nodes) {
+ StartElement element = node.getStartElement();
+ int line = element.getLocation().getLineNumber();
+
+ if (!element.getName().equals(new QName("property"))) {
+ errors.add(String.format("Line %d: element not <property>", line));
+ continue;
+ }
+
+ List<XMLEvent> events = node.getXMLEventsForQName(new QName("name"));
+ if (events == null) {
+ errors.add(String.format("Line %d: <property> has no <name>", line));
+ } else {
+ String v = null;
+ for (XMLEvent event : events) {
+ if (event.isAttribute()) {
+ v = ((Attribute) event).getValue();
+ } else {
+ Characters c = node.getElement(event.asStartElement());
+ if (c != null) {
+ v = c.getData();
+ }
+ }
+ if (v == null || v.isEmpty()) {
+ errors.add(String.format("Line %d: <property> has an empty <name>",
+ line));
+ }
+ }
+ if (v != null && !v.isEmpty()) {
+ List<Integer> lines = duplicatedProperties.get(v);
+ if (lines == null) {
+ lines = new ArrayList<Integer>();
+ duplicatedProperties.put(v, lines);
+ }
+ lines.add(node.getStartElement().getLocation().getLineNumber());
+ }
+ }
+
+ events = node.getXMLEventsForQName(new QName("value"));
+ if (events == null) {
+ errors.add(String.format("Line %d: <property> has no <value>", line));
+ }
+
+ for (QName qName : node.getDuplicatedQNames()) {
+ if (!qName.equals(new QName("source"))) {
+ errors.add(String.format("Line %d: <property> has duplicated <%s>s",
+ line, qName));
+ }
+ }
+ }
+
+ for (Entry<String, List<Integer>> e : duplicatedProperties.entrySet()) {
+ List<Integer> lines = e.getValue();
+ if (1 < lines.size()) {
+ errors.add(String.format("Line %s: duplicated <property>s for %s",
+ StringUtils.join(", ", lines), e.getKey()));
+ }
+ }
+
+ return errors;
+ }
+
+ private static File[] listFiles(File dir) {
+ return dir.listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File file) {
+ return file.isFile() && file.getName().endsWith(".xml");
+ }
+ });
+ }
+
+ @SuppressWarnings("static-access")
+ public static void main(String[] args) throws IOException {
+ GenericOptionsParser genericParser = new GenericOptionsParser(args);
+ String[] remainingArgs = genericParser.getRemainingArgs();
+
+ Option conf = OptionBuilder.hasArg().create("conffile");
+ Option help = OptionBuilder.withLongOpt("help").create('h');
+ Options opts = new Options().addOption(conf).addOption(help);
+ CommandLineParser specificParser = new GnuParser();
+ CommandLine cmd = null;
+ try {
+ cmd = specificParser.parse(opts, remainingArgs);
+ } catch (MissingArgumentException e) {
+ terminate(1, "No argument specified for -conffile option");
+ } catch (ParseException e) {
+ terminate(1, USAGE);
+ }
+ if (cmd == null) {
+ terminate(1, "Failed to parse options");
+ }
+
+ if (cmd.hasOption('h')) {
+ terminate(0, USAGE);
+ }
+
+ List<File> files = new ArrayList<File>();
+ if (cmd.hasOption("conffile")) {
+ String[] values = cmd.getOptionValues("conffile");
+ for (String value : values) {
+ File confFile = new File(value);
+ if (confFile.isFile()) {
+ files.add(confFile);
+ } else if (confFile.isDirectory()) {
+ for (File file : listFiles(confFile)) {
+ files.add(file);
+ }
+ } else {
+ terminate(1, confFile.getAbsolutePath()
+ + " is neither a file nor directory");
+ }
+ }
+ } else {
+ String confDirName = System.getenv(HADOOP_CONF_DIR);
+ if (confDirName == null) {
+ terminate(1, HADOOP_CONF_DIR + " does not defined");
+ }
+ File confDir = new File(confDirName);
+ if (!confDir.isDirectory()) {
+ terminate(1, HADOOP_CONF_DIR + " is not a directory");
+ }
+ files = Arrays.asList(listFiles(confDir));
+ }
+ if (files.isEmpty()) {
+ terminate(1, "No input file to validate");
+ }
+
+ boolean ok = true;
+ for (File file : files) {
+ String path = file.getAbsolutePath();
+ List<String> errors = checkConf(new FileInputStream(file));
+ if (errors.isEmpty()) {
+ System.out.println(path + ": valid");
+ } else {
+ ok = false;
+ System.err.println(path + ":");
+ for (String error : errors) {
+ System.err.println("\t" + error);
+ }
+ }
+ }
+ if (ok) {
+ System.out.println("OK");
+ } else {
+ terminate(1, "Invalid file exists");
+ }
+ }
+
+ private static void terminate(int status, String msg) {
+ System.err.println(msg);
+ System.exit(status);
+ }
+}
+
+class NodeInfo {
+
+ private StartElement startElement;
+ private List<Attribute> attributes = new ArrayList<Attribute>();
+ private Map<StartElement, Characters> elements =
+ new HashMap<StartElement, Characters>();
+ private Map<QName, List<XMLEvent>> qNameXMLEventsMap =
+ new HashMap<QName, List<XMLEvent>>();
+
+ public NodeInfo(StartElement startElement) {
+ this.startElement = startElement;
+ }
+
+ private void addQNameXMLEvent(QName qName, XMLEvent event) {
+ List<XMLEvent> events = qNameXMLEventsMap.get(qName);
+ if (events == null) {
+ events = new ArrayList<XMLEvent>();
+ qNameXMLEventsMap.put(qName, events);
+ }
+ events.add(event);
+ }
+
+ public StartElement getStartElement() {
+ return startElement;
+ }
+
+ public void addAttribute(Attribute attribute) {
+ attributes.add(attribute);
+ addQNameXMLEvent(attribute.getName(), attribute);
+ }
+
+ public Characters getElement(StartElement element) {
+ return elements.get(element);
+ }
+
+ public void addElement(StartElement element) {
+ setElement(element, null);
+ addQNameXMLEvent(element.getName(), element);
+ }
+
+ public void setElement(StartElement element, Characters text) {
+ elements.put(element, text);
+ }
+
+ public List<QName> getDuplicatedQNames() {
+ List<QName> duplicates = new ArrayList<QName>();
+ for (Map.Entry<QName, List<XMLEvent>> e : qNameXMLEventsMap.entrySet()) {
+ if (1 < e.getValue().size()) {
+ duplicates.add(e.getKey());
+ }
+ }
+ return duplicates;
+ }
+
+ public List<XMLEvent> getXMLEventsForQName(QName qName) {
+ return qNameXMLEventsMap.get(qName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f47c0e9a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfTest.java
new file mode 100644
index 0000000..f6cc7c3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestConfTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayInputStream;
+import java.util.List;
+
+import org.junit.Test;
+
+public class TestConfTest {
+
+ @Test
+ public void testEmptyConfiguration() {
+ String conf = "<configuration/>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertTrue(errors.isEmpty());
+ }
+
+ @Test
+ public void testValidConfiguration() {
+ String conf = "<configuration>\n"
+ + "<property>\n"
+ + "<name>foo</name>\n"
+ + "<value>bar</value>\n"
+ + "</property>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertTrue(errors.isEmpty());
+ }
+
+ @Test
+ public void testSourceDuplicationIsValid() {
+ String conf = "<configuration>\n"
+ + "<property source='a'>\n"
+ + "<name>foo</name>\n"
+ + "<value>bar</value>\n"
+ + "<source>b</source>\n"
+ + "<source>c</source>\n"
+ + "</property>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertTrue(errors.isEmpty());
+ }
+
+ @Test
+ public void testEmptyInput() {
+ String conf = "";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertTrue(errors.get(0).startsWith("bad conf file: "));
+ }
+
+ @Test
+ public void testInvalidFormat() {
+ String conf = "<configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertTrue(errors.get(0).startsWith("bad conf file: "));
+ }
+
+ @Test
+ public void testRootElementNotConfiguration() {
+ String conf = "<configurations/>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertEquals("bad conf file: top-level element not <configuration>", errors.get(0));
+ }
+
+ @Test
+ public void testSubElementNotProperty() {
+ String conf = "<configuration>\n"
+ + "<foo/>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertEquals("Line 2: element not <property>", errors.get(0));
+ }
+
+ @Test
+ public void testPropertyHasNoName() {
+ String conf ="<configuration>\n"
+ + "<property>\n"
+ + "<value>foo</value>\n"
+ + "</property>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertEquals("Line 2: <property> has no <name>", errors.get(0));
+ }
+
+ @Test
+ public void testPropertyHasEmptyName() {
+ String conf = "<configuration>\n"
+ + "<property>\n"
+ + "<name></name>\n"
+ + "<value>foo</value>\n"
+ + "</property>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertEquals("Line 2: <property> has an empty <name>", errors.get(0));
+ }
+
+ @Test
+ public void testPropertyHasNoValue() {
+ String conf ="<configuration>\n"
+ + "<property>\n"
+ + "<name>foo</name>\n"
+ + "</property>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertEquals("Line 2: <property> has no <value>", errors.get(0));
+ }
+
+ @Test
+ public void testPropertyHasEmptyValue() {
+ String conf = "<configuration>\n"
+ + "<property>\n"
+ + "<name>foo</name>\n"
+ + "<value></value>\n"
+ + "</property>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertTrue(errors.isEmpty());
+ }
+
+ @Test
+ public void testPropertyHasDuplicatedAttributeAndElement() {
+ String conf = "<configuration>\n"
+ + "<property name='foo'>\n"
+ + "<name>bar</name>\n"
+ + "<value>baz</value>\n"
+ + "</property>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertEquals("Line 2: <property> has duplicated <name>s", errors.get(0));
+ }
+
+ @Test
+ public void testPropertyHasDuplicatedElements() {
+ String conf = "<configuration>\n"
+ + "<property>\n"
+ + "<name>foo</name>\n"
+ + "<name>bar</name>\n"
+ + "<value>baz</value>\n"
+ + "</property>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertEquals("Line 2: <property> has duplicated <name>s", errors.get(0));
+ }
+
+ @Test
+ public void testDuplicatedProperties() {
+ String conf = "<configuration>\n"
+ + "<property>\n"
+ + "<name>foo</name>\n"
+ + "<value>bar</value>\n"
+ + "</property>\n"
+ + "<property>\n"
+ + "<name>foo</name>\n"
+ + "<value>baz</value>\n"
+ + "</property>\n"
+ + "</configuration>";
+ ByteArrayInputStream bais = new ByteArrayInputStream(conf.getBytes());
+ List<String> errors = ConfTest.checkConf(bais);
+ assertEquals(1, errors.size());
+ assertEquals("Line 2, 6: duplicated <property>s for foo", errors.get(0));
+ }
+
+}
[47/50] [abbrv] hadoop git commit: HADOOP-12037. Fix wrong classname
in example configuration of hadoop-auth documentation. Contributed by
Masatake Iwasaki.
Posted by zj...@apache.org.
HADOOP-12037. Fix wrong classname in example configuration of hadoop-auth documentation. Contributed by Masatake Iwasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77c1692d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77c1692d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77c1692d
Branch: refs/heads/YARN-2928
Commit: 77c1692d010ad893bf40805192ac0aa9aa6049d2
Parents: 065ab4d
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Jun 1 18:04:52 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:44:22 2015 -0700
----------------------------------------------------------------------
.../src/site/markdown/Configuration.md | 18 +++++++++++++++---
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
2 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77c1692d/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md b/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
index 9d076bb..2f9b860 100644
--- a/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
+++ b/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
@@ -73,12 +73,13 @@ To use Kerberos SPNEGO as the authentication mechanism, the authentication filte
**Example**:
+```xml
<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
...
<filter>
<filter-name>kerberosFilter</filter-name>
- <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
+ <filter-class>org.apache.hadoop.security.authentication.server.AuthenticationFilter</filter-class>
<init-param>
<param-name>type</param-name>
<param-value>kerberos</param-value>
@@ -112,6 +113,7 @@ To use Kerberos SPNEGO as the authentication mechanism, the authentication filte
...
</web-app>
+```
### Pseudo/Simple Configuration
@@ -125,12 +127,13 @@ To use Pseudo/Simple as the authentication mechanism (trusting the value of the
**Example**:
+```xml
<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
...
<filter>
<filter-name>simpleFilter</filter-name>
- <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
+ <filter-class>org.apache.hadoop.security.authentication.server.AuthenticationFilter</filter-class>
<init-param>
<param-name>type</param-name>
<param-value>simple</param-value>
@@ -160,6 +163,7 @@ To use Pseudo/Simple as the authentication mechanism (trusting the value of the
...
</web-app>
+```
### AltKerberos Configuration
@@ -175,12 +179,13 @@ The AltKerberos authentication mechanism is a partially implemented derivative o
**Example**:
+```xml
<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
...
<filter>
<filter-name>kerberosFilter</filter-name>
- <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
+ <filter-class>org.apache.hadoop.security.authentication.server.AuthenticationFilter</filter-class>
<init-param>
<param-name>type</param-name>
<param-value>org.my.subclass.of.AltKerberosAuthenticationHandler</param-value>
@@ -218,6 +223,7 @@ The AltKerberos authentication mechanism is a partially implemented derivative o
...
</web-app>
+```
### SignerSecretProvider Configuration
@@ -262,6 +268,7 @@ The following configuration properties are specific to the `zookeeper` implement
**Example**:
+```xml
<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
...
@@ -279,9 +286,11 @@ The following configuration properties are specific to the `zookeeper` implement
...
</web-app>
+```
**Example**:
+```xml
<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
...
@@ -299,9 +308,11 @@ The following configuration properties are specific to the `zookeeper` implement
...
</web-app>
+```
**Example**:
+```xml
<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
...
@@ -339,3 +350,4 @@ The following configuration properties are specific to the `zookeeper` implement
...
</web-app>
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77c1692d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 44c2969..a0e6e90 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -631,6 +631,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12043. Display warning if defaultFs is not set when running fs
commands. (Lei Xu via wang)
+ HADOOP-12037. Fix wrong classname in example configuration of hadoop-auth
+ documentation. (Masatake Iwasaki via wang)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
[07/50] [abbrv] hadoop git commit: YARN-3722. Merge multiple
TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils. Contributed by
Masatake Iwasaki.
Posted by zj...@apache.org.
YARN-3722. Merge multiple TestWebAppUtils into
o.a.h.yarn.webapp.util.TestWebAppUtils. Contributed by Masatake Iwasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/822cac2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/822cac2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/822cac2a
Branch: refs/heads/YARN-2928
Commit: 822cac2a90637b221d7c4357f8630167223bd982
Parents: 2b39cff
Author: Devaraj K <de...@apache.org>
Authored: Thu May 28 15:35:06 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 16:12:53 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/util/TestWebAppUtils.java | 81 --------------------
.../yarn/webapp/util/TestWebAppUtils.java | 52 +++++++++++++
3 files changed, 55 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822cac2a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5ce8c63..5326f2d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -365,6 +365,9 @@ Release 2.8.0 - UNRELEASED
YARN-3700. Made generic history service load a number of latest applications
according to the parameter or the configuration. (Xuan Gong via zjshen)
+ YARN-3722. Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils.
+ (Masatake Iwasaki via devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822cac2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
deleted file mode 100644
index 169787f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class TestWebAppUtils {
-
- private static final String RM1_NODE_ID = "rm1";
- private static final String RM2_NODE_ID = "rm2";
-
- // Because WebAppUtils#getResolvedAddress tries to resolve the hostname, we add a static mapping for dummy hostnames
- // to make this test run anywhere without having to give some resolvable hostnames
- private static String dummyHostNames[] = {"host1", "host2", "host3"};
- private static final String anyIpAddress = "1.2.3.4";
- private static Map<String, String> savedStaticResolution = new HashMap<>();
-
- @BeforeClass
- public static void initializeDummyHostnameResolution() throws Exception {
- String previousIpAddress;
- for (String hostName : dummyHostNames) {
- if (null != (previousIpAddress = NetUtils.getStaticResolution(hostName))) {
- savedStaticResolution.put(hostName, previousIpAddress);
- }
- NetUtils.addStaticResolution(hostName, anyIpAddress);
- }
- }
-
- @AfterClass
- public static void restoreDummyHostnameResolution() throws Exception {
- for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution.entrySet()) {
- NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), hostnameToIpEntry.getValue());
- }
- }
-
- @Test
- public void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
- Configuration configuration = new Configuration();
- final String rmAddress = "host1:8088";
- configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, rmAddress);
- final String rm1Address = "host2:8088";
- final String rm2Address = "host3:8088";
- configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID, rm1Address);
- configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address);
- configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
- configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
-
- String rmRemoteUrl = WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(configuration);
- Assert.assertEquals("ResolvedRemoteRMWebAppUrl should resolve to the first HA RM address", rm1Address, rmRemoteUrl);
-
- String rmLocalUrl = WebAppUtils.getResolvedRMWebAppURLWithoutScheme(configuration);
- Assert.assertEquals("ResolvedRMWebAppUrl should resolve to the default RM webapp address", rmAddress, rmLocalUrl);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822cac2a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
index 2bd91b4..dcc8ba4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/util/TestWebAppUtils.java
@@ -23,17 +23,69 @@ import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.http.HttpServer2.Builder;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Assert;
import org.junit.Test;
public class TestWebAppUtils {
+ private static final String RM1_NODE_ID = "rm1";
+ private static final String RM2_NODE_ID = "rm2";
+
+ // Because WebAppUtils#getResolvedAddress tries to resolve the hostname, we add a static mapping for dummy hostnames
+ // to make this test run anywhere without having to give some resolvable hostnames
+ private static String dummyHostNames[] = {"host1", "host2", "host3"};
+ private static final String anyIpAddress = "1.2.3.4";
+ private static Map<String, String> savedStaticResolution = new HashMap<>();
+
+ @BeforeClass
+ public static void initializeDummyHostnameResolution() throws Exception {
+ String previousIpAddress;
+ for (String hostName : dummyHostNames) {
+ if (null != (previousIpAddress = NetUtils.getStaticResolution(hostName))) {
+ savedStaticResolution.put(hostName, previousIpAddress);
+ }
+ NetUtils.addStaticResolution(hostName, anyIpAddress);
+ }
+ }
+
+ @AfterClass
+ public static void restoreDummyHostnameResolution() throws Exception {
+ for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution.entrySet()) {
+ NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), hostnameToIpEntry.getValue());
+ }
+ }
+
+ @Test
+ public void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
+ Configuration configuration = new Configuration();
+ final String rmAddress = "host1:8088";
+ configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, rmAddress);
+ final String rm1Address = "host2:8088";
+ final String rm2Address = "host3:8088";
+ configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID, rm1Address);
+ configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address);
+ configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
+
+ String rmRemoteUrl = WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(configuration);
+ Assert.assertEquals("ResolvedRemoteRMWebAppUrl should resolve to the first HA RM address", rm1Address, rmRemoteUrl);
+
+ String rmLocalUrl = WebAppUtils.getResolvedRMWebAppURLWithoutScheme(configuration);
+ Assert.assertEquals("ResolvedRMWebAppUrl should resolve to the default RM webapp address", rmAddress, rmLocalUrl);
+ }
@Test
public void testGetPassword() throws Exception {