You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/05/19 19:46:50 UTC
[01/50] [abbrv] hadoop git commit: HDFS-10333. Intermittent
org.apache.hadoop.hdfs.TestFileAppend failure in trunk. Contributed by Yiqun
Lin.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 4770ae7e3 -> 76279f587
HDFS-10333. Intermittent org.apache.hadoop.hdfs.TestFileAppend failure in trunk. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45788204
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45788204
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45788204
Branch: refs/heads/HDFS-7240
Commit: 45788204ae2ac82ccb3b4fe2fd22aead1dd79f0d
Parents: f45bc5a
Author: Andrew Wang <wa...@apache.org>
Authored: Sun May 15 22:15:05 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Sun May 15 22:15:05 2016 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45788204/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
index 2768d21..3e8561f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.HardLink;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -371,6 +372,9 @@ public class TestFileAppend{
final Configuration conf = new HdfsConfiguration();
conf.setInt(
DFSConfigKeys.DFS_NAMENODE_FILE_CLOSE_NUM_COMMITTED_ALLOWED_KEY, 1);
+ conf.setBoolean(
+ HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.ENABLE_KEY,
+ false);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(4).build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDFS-10208. Addendum for
HDFS-9579: to handle the case when client machine can't resolve network path
(Ming Ma via sjlee)
Posted by ar...@apache.org.
HDFS-10208. Addendum for HDFS-9579: to handle the case when client machine can't resolve network path (Ming Ma via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61f46be0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61f46be0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61f46be0
Branch: refs/heads/HDFS-7240
Commit: 61f46be071e42f9eb49a54b1bd2e54feac59f808
Parents: 730bc74
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon May 16 18:49:47 2016 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Mon May 16 18:49:47 2016 -0700
----------------------------------------------------------------------
.../fs/CommonConfigurationKeysPublic.java | 7 ++-
.../org/apache/hadoop/net/NetworkTopology.java | 35 ++++++++++++
.../java/org/apache/hadoop/net/NodeBase.java | 9 ++++
.../src/main/resources/core-default.xml | 13 +++++
.../org/apache/hadoop/hdfs/ClientContext.java | 47 +++++++++-------
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 31 +++++++----
.../hadoop/hdfs/TestDistributedFileSystem.java | 57 +++++++++++++++++---
.../apache/hadoop/net/TestNetworkTopology.java | 9 ++++
8 files changed, 173 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61f46be0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 648ad59..f6ccc56 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -111,7 +111,12 @@ public class CommonConfigurationKeysPublic {
public static final String FS_TRASH_INTERVAL_KEY = "fs.trash.interval";
/** Default value for FS_TRASH_INTERVAL_KEY */
public static final long FS_TRASH_INTERVAL_DEFAULT = 0;
-
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a>. */
+ public static final String FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED =
+ "fs.client.resolve.topology.enabled";
+ /** Default value for FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED. */
+ public static final boolean FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT =
+ false;
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String IO_MAPFILE_BLOOM_SIZE_KEY =
"io.mapfile.bloom.size";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61f46be0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 1e23ff6..cf5b176 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -655,6 +655,41 @@ public class NetworkTopology {
return dis+2;
}
+ /** Return the distance between two nodes by comparing their network paths
+ * without checking if they belong to the same ancestor node by reference.
+ * It is assumed that the distance from one node to its parent is 1
+ * The distance between two nodes is calculated by summing up their distances
+ * to their closest common ancestor.
+ * @param node1 one node
+ * @param node2 another node
+ * @return the distance between node1 and node2
+ */
+ static public int getDistanceByPath(Node node1, Node node2) {
+ if (node1 == null && node2 == null) {
+ return 0;
+ }
+ if (node1 == null || node2 == null) {
+ LOG.warn("One of the nodes is a null pointer");
+ return Integer.MAX_VALUE;
+ }
+ String[] paths1 = NodeBase.getPathComponents(node1);
+ String[] paths2 = NodeBase.getPathComponents(node2);
+ int dis = 0;
+ int index = 0;
+ int minLevel = Math.min(paths1.length, paths2.length);
+ while (index < minLevel) {
+ if (!paths1[index].equals(paths2[index])) {
+ // Once the path starts to diverge, compute the distance that include
+ // the rest of paths.
+ dis += 2 * (minLevel - index);
+ break;
+ }
+ index++;
+ }
+ dis += Math.abs(paths1.length - paths2.length);
+ return dis;
+ }
+
/** Check if two nodes are on the same rack
* @param node1 one node (can be null)
* @param node2 another node (can be null)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61f46be0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
index b465098..7494adf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
@@ -113,6 +113,15 @@ public class NodeBase implements Node {
return node.getNetworkLocation() + PATH_SEPARATOR_STR + node.getName();
}
+ /**
+ * Get the path components of a node.
+ * @param node a non-null node
+ * @return the path of a node
+ */
+ public static String[] getPathComponents(Node node) {
+ return getPath(node).split(PATH_SEPARATOR_STR);
+ }
+
@Override
public boolean equals(Object to) {
if (this == to) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61f46be0/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 2b13133..0998b0e 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2143,4 +2143,17 @@
<name>hadoop.http.logs.enabled</name>
<value>true</value>
</property>
+
+ <property>
+ <name>fs.client.resolve.topology.enabled</name>
+ <value>false</value>
+ <description>Whether the client machine will use the class specified by
+ property net.topology.node.switch.mapping.impl to compute the network
+ distance between itself and remote machines of the FileSystem. Additional
+ properties might need to be configured depending on the class specified
+ in net.topology.node.switch.mapping.impl. For example, if
+ org.apache.hadoop.net.ScriptBasedMapping is used, a valid script file
+ needs to be specified in net.topology.script.file.name.
+ </description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61f46be0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
index 47d6d49..a31945c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
@@ -17,11 +17,12 @@
*/
package org.apache.hadoop.hdfs;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -113,9 +114,8 @@ public class ClientContext {
*/
private boolean printedConfWarning = false;
- private final NetworkTopology topology;
- private final NodeBase clientNode;
- private final Map<NodeBase, Integer> nodeToDistance;
+ private NodeBase clientNode;
+ private boolean topologyResolutionEnabled;
private ClientContext(String name, DfsClientConf conf,
Configuration config) {
@@ -133,19 +133,29 @@ public class ClientContext {
this.byteArrayManager = ByteArrayManager.newInstance(
conf.getWriteByteArrayManagerConf());
+ initTopologyResolution(config);
+ }
+ private void initTopologyResolution(Configuration config) {
+ topologyResolutionEnabled = config.getBoolean(
+ FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED,
+ FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED_DEFAULT);
+ if (!topologyResolutionEnabled) {
+ return;
+ }
DNSToSwitchMapping dnsToSwitchMapping = ReflectionUtils.newInstance(
config.getClass(
CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
ScriptBasedMapping.class, DNSToSwitchMapping.class), config);
- List<String> nodes = new ArrayList<>();
String clientHostName = NetUtils.getLocalHostname();
+ List<String> nodes = new ArrayList<>();
nodes.add(clientHostName);
- clientNode = new NodeBase(clientHostName,
- dnsToSwitchMapping.resolve(nodes).get(0));
- this.topology = NetworkTopology.getInstance(config);
- this.topology.add(clientNode);
- this.nodeToDistance = new ConcurrentHashMap<>();
+ List<String> resolvedHosts = dnsToSwitchMapping.resolve(nodes);
+ if (resolvedHosts != null && !resolvedHosts.isEmpty() &&
+ !resolvedHosts.get(0).equals(NetworkTopology.DEFAULT_RACK)) {
+ // The client machine is able to resolve its own network location.
+ this.clientNode = new NodeBase(clientHostName, resolvedHosts.get(0));
+ }
}
public static ClientContext get(String name, DfsClientConf conf,
@@ -229,14 +239,15 @@ public class ClientContext {
}
public int getNetworkDistance(DatanodeInfo datanodeInfo) {
+ // If applications disable the feature or the client machine can't
+ // resolve its network location, clientNode will be set to null.
+ if (clientNode == null) {
+ return DFSUtilClient.isLocalAddress(NetUtils.
+ createSocketAddr(datanodeInfo.getXferAddr())) ? 0 :
+ Integer.MAX_VALUE;
+ }
NodeBase node = new NodeBase(datanodeInfo.getHostName(),
datanodeInfo.getNetworkLocation());
- Integer distance = nodeToDistance.get(node);
- if (distance == null) {
- topology.add(node);
- distance = topology.getDistance(clientNode, node);
- nodeToDistance.put(node, distance);
- }
- return distance;
+ return NetworkTopology.getDistanceByPath(clientNode, node);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61f46be0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 2eff12e..9359023 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -189,7 +189,8 @@ public class MiniDFSCluster {
private boolean checkDataNodeHostConfig = false;
private Configuration[] dnConfOverlays;
private boolean skipFsyncForTesting = true;
-
+ private boolean useConfiguredTopologyMappingClass = false;
+
public Builder(Configuration conf) {
this.conf = conf;
this.storagesPerDatanode =
@@ -433,7 +434,14 @@ public class MiniDFSCluster {
this.skipFsyncForTesting = val;
return this;
}
-
+
+ public Builder useConfiguredTopologyMappingClass(
+ boolean useConfiguredTopologyMappingClass) {
+ this.useConfiguredTopologyMappingClass =
+ useConfiguredTopologyMappingClass;
+ return this;
+ }
+
/**
* Construct the actual MiniDFSCluster
*/
@@ -501,7 +509,8 @@ public class MiniDFSCluster {
builder.checkDataNodeAddrConfig,
builder.checkDataNodeHostConfig,
builder.dnConfOverlays,
- builder.skipFsyncForTesting);
+ builder.skipFsyncForTesting,
+ builder.useConfiguredTopologyMappingClass);
}
public class DataNodeProperties {
@@ -756,12 +765,13 @@ public class MiniDFSCluster {
operation, null, racks, hosts,
null, simulatedCapacities, null, true, false,
MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0),
- true, false, false, null, true);
+ true, false, false, null, true, false);
}
private void initMiniDFSCluster(
Configuration conf,
- int numDataNodes, StorageType[][] storageTypes, boolean format, boolean manageNameDfsDirs,
+ int numDataNodes, StorageType[][] storageTypes, boolean format,
+ boolean manageNameDfsDirs,
boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy,
boolean manageDataDfsDirs, StartupOption startOpt,
StartupOption dnStartOpt, String[] racks,
@@ -772,7 +782,8 @@ public class MiniDFSCluster {
boolean checkDataNodeAddrConfig,
boolean checkDataNodeHostConfig,
Configuration[] dnConfOverlays,
- boolean skipFsyncForTesting)
+ boolean skipFsyncForTesting,
+ boolean useConfiguredTopologyMappingClass)
throws IOException {
boolean success = false;
try {
@@ -797,9 +808,11 @@ public class MiniDFSCluster {
DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY, 0);
conf.setInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, safemodeExtension);
conf.setInt(DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 3); // 3 second
- conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
- StaticMapping.class, DNSToSwitchMapping.class);
-
+ if (!useConfiguredTopologyMappingClass) {
+ conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+ StaticMapping.class, DNSToSwitchMapping.class);
+ }
+
// In an HA cluster, in order for the StandbyNode to perform checkpoints,
// it needs to know the HTTP port of the Active. So, if ephemeral ports
// are chosen, disable checkpoints for the test.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61f46be0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index f3bf6b5..ac960c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -80,7 +82,10 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.Op;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.net.DNSToSwitchMapping;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ScriptBasedMapping;
+import org.apache.hadoop.net.StaticMapping;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DataChecksum;
@@ -798,39 +803,77 @@ public class TestDistributedFileSystem {
@Test
public void testLocalHostReadStatistics() throws Exception {
- testReadFileSystemStatistics(0);
+ testReadFileSystemStatistics(0, false, false);
}
@Test
public void testLocalRackReadStatistics() throws Exception {
- testReadFileSystemStatistics(2);
+ testReadFileSystemStatistics(2, false, false);
}
@Test
public void testRemoteRackOfFirstDegreeReadStatistics() throws Exception {
- testReadFileSystemStatistics(4);
+ testReadFileSystemStatistics(4, false, false);
+ }
+
+ @Test
+ public void testInvalidScriptMappingFileReadStatistics() throws Exception {
+ // Even though network location of the client machine is unknown,
+ // MiniDFSCluster's datanode is on the local host and thus the network
+ // distance is 0.
+ testReadFileSystemStatistics(0, true, true);
+ }
+
+ @Test
+ public void testEmptyScriptMappingFileReadStatistics() throws Exception {
+ // Network location of the client machine is resolved to
+ // {@link NetworkTopology#DEFAULT_RACK} when there is no script file
+ // defined. This is equivalent to unknown network location.
+ // MiniDFSCluster's datanode is on the local host and thus the network
+ // distance is 0.
+ testReadFileSystemStatistics(0, true, false);
}
/** expectedDistance is the expected distance between client and dn.
* 0 means local host.
* 2 means same rack.
* 4 means remote rack of first degree.
+ * invalidScriptMappingConfig is used to test
*/
- private void testReadFileSystemStatistics(int expectedDistance)
+ private void testReadFileSystemStatistics(int expectedDistance,
+ boolean useScriptMapping, boolean invalidScriptMappingFile)
throws IOException {
MiniDFSCluster cluster = null;
+ StaticMapping.addNodeToRack(NetUtils.getLocalHostname(), "/rackClient");
final Configuration conf = getTestConfiguration();
+ conf.setBoolean(FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED, true);
+ // ClientContext is cached globally by default thus we will end up using
+ // the network distance computed by other test cases.
+ // Use different value for DFS_CLIENT_CONTEXT in each test case so that it
+ // can compute network distance independently.
+ conf.set(DFS_CLIENT_CONTEXT, "testContext_" + expectedDistance);
// create a cluster with a dn with the expected distance.
- if (expectedDistance == 0) {
+ // MiniDFSCluster by default uses StaticMapping unless the test
+ // overrides it.
+ if (useScriptMapping) {
+ conf.setClass(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+ ScriptBasedMapping.class, DNSToSwitchMapping.class);
+ if (invalidScriptMappingFile) {
+ conf.set(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY,
+ "invalidScriptFile.txt");
+ }
+ cluster = new MiniDFSCluster.Builder(conf).
+ useConfiguredTopologyMappingClass(true).build();
+ } else if (expectedDistance == 0) {
cluster = new MiniDFSCluster.Builder(conf).
hosts(new String[] {NetUtils.getLocalHostname()}).build();
} else if (expectedDistance == 2) {
cluster = new MiniDFSCluster.Builder(conf).
- hosts(new String[] {"hostFoo"}).build();
+ racks(new String[]{"/rackClient"}).build();
} else if (expectedDistance == 4) {
cluster = new MiniDFSCluster.Builder(conf).
- racks(new String[] {"/rackFoo"}).build();
+ racks(new String[]{"/rackFoo"}).build();
}
// create a file, read the file and verify the metrics
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61f46be0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
index 7285d1b..d149f65 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
@@ -144,6 +144,15 @@ public class TestNetworkTopology {
NodeBase node2 = new NodeBase(dataNodes[0].getHostName(),
dataNodes[0].getNetworkLocation());
assertEquals(0, cluster.getDistance(node1, node2));
+ // verify the distance can be computed by path.
+ // They don't need to refer to the same object or parents.
+ NodeBase node3 = new NodeBase(dataNodes[3].getHostName(),
+ dataNodes[3].getNetworkLocation());
+ NodeBase node4 = new NodeBase(dataNodes[6].getHostName(),
+ dataNodes[6].getNetworkLocation());
+ assertEquals(0, NetworkTopology.getDistanceByPath(node1, node2));
+ assertEquals(4, NetworkTopology.getDistanceByPath(node2, node3));
+ assertEquals(6, NetworkTopology.getDistanceByPath(node2, node4));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDFS-10360. DataNode may format
directory and lose blocks if current/VERSION is missing. (Wei-Chiu Chuang via
lei)
Posted by ar...@apache.org.
HDFS-10360. DataNode may format directory and lose blocks if current/VERSION is missing. (Wei-Chiu Chuang via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf552aa8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf552aa8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf552aa8
Branch: refs/heads/HDFS-7240
Commit: cf552aa87b4c47f0c73f51f44f3bc1d267c524cf
Parents: f4d8fde
Author: Lei Xu <le...@apache.org>
Authored: Tue May 17 16:36:48 2016 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Wed May 18 09:38:37 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/common/Storage.java | 65 ++++++++++++++++++--
.../hdfs/server/datanode/DataStorage.java | 2 +-
.../TestDataNodeVolumeFailureReporting.java | 36 +++++++++++
.../hdfs/server/datanode/TestDataStorage.java | 26 ++++++++
4 files changed, 124 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf552aa8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index 41719b9..9218e9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -25,7 +25,10 @@ import java.io.RandomAccessFile;
import java.lang.management.ManagementFactory;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
@@ -345,9 +348,12 @@ public abstract class Storage extends StorageInfo {
*/
public void clearDirectory() throws IOException {
File curDir = this.getCurrentDir();
- if (curDir.exists())
+ if (curDir.exists()) {
+ File[] files = FileUtil.listFiles(curDir);
+ LOG.info("Will remove files: " + Arrays.toString(files));
if (!(FileUtil.fullyDelete(curDir)))
throw new IOException("Cannot remove current directory: " + curDir);
+ }
if (!curDir.mkdirs())
throw new IOException("Cannot create directory " + curDir);
}
@@ -470,16 +476,60 @@ public abstract class Storage extends StorageInfo {
}
/**
- * Check consistency of the storage directory
+ * Check to see if current/ directory is empty. This method is used
+ * before determining to format the directory.
+ *
+ * @throws InconsistentFSStateException if not empty.
+ * @throws IOException if unable to list files under the directory.
+ */
+ private void checkEmptyCurrent() throws InconsistentFSStateException,
+ IOException {
+ File currentDir = getCurrentDir();
+ if(!currentDir.exists()) {
+ // if current/ does not exist, it's safe to format it.
+ return;
+ }
+ try(DirectoryStream<java.nio.file.Path> dirStream =
+ Files.newDirectoryStream(currentDir.toPath())) {
+ if (dirStream.iterator().hasNext()) {
+ throw new InconsistentFSStateException(root,
+ "Can't format the storage directory because the current/ "
+ + "directory is not empty.");
+ }
+ }
+ }
+
+ /**
+ * Check consistency of the storage directory.
+ *
+ * @param startOpt a startup option.
+ * @param storage The Storage object that manages this StorageDirectory.
+ *
+ * @return state {@link StorageState} of the storage directory
+ * @throws InconsistentFSStateException if directory state is not
+ * consistent and cannot be recovered.
+ * @throws IOException
+ */
+ public StorageState analyzeStorage(StartupOption startOpt, Storage storage)
+ throws IOException {
+ return analyzeStorage(startOpt, storage, false);
+ }
+
+ /**
+ * Check consistency of the storage directory.
*
* @param startOpt a startup option.
+ * @param storage The Storage object that manages this StorageDirectory.
+ * @param checkCurrentIsEmpty if true, make sure current/ directory
+ * is empty before determining to format it.
*
* @return state {@link StorageState} of the storage directory
* @throws InconsistentFSStateException if directory state is not
* consistent and cannot be recovered.
* @throws IOException
*/
- public StorageState analyzeStorage(StartupOption startOpt, Storage storage)
+ public StorageState analyzeStorage(StartupOption startOpt, Storage storage,
+ boolean checkCurrentIsEmpty)
throws IOException {
assert root != null : "root is null";
boolean hadMkdirs = false;
@@ -516,8 +566,12 @@ public abstract class Storage extends StorageInfo {
// If startOpt is HOTSWAP, it returns NOT_FORMATTED for empty directory,
// while it also checks the layout version.
if (startOpt == HdfsServerConstants.StartupOption.FORMAT ||
- (startOpt == StartupOption.HOTSWAP && hadMkdirs))
+ (startOpt == StartupOption.HOTSWAP && hadMkdirs)) {
+ if (checkCurrentIsEmpty) {
+ checkEmptyCurrent();
+ }
return StorageState.NOT_FORMATTED;
+ }
if (startOpt != HdfsServerConstants.StartupOption.IMPORT) {
storage.checkOldLayoutStorage(this);
@@ -542,6 +596,9 @@ public abstract class Storage extends StorageInfo {
if (hasPrevious)
throw new InconsistentFSStateException(root,
"version file in current directory is missing.");
+ if (checkCurrentIsEmpty) {
+ checkEmptyCurrent();
+ }
return StorageState.NOT_FORMATTED;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf552aa8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
index 3844c8e..0e6b339 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
@@ -267,7 +267,7 @@ public class DataStorage extends Storage {
List<Callable<StorageDirectory>> callables) throws IOException {
StorageDirectory sd = new StorageDirectory(dataDir, null, false);
try {
- StorageState curState = sd.analyzeStorage(startOpt, this);
+ StorageState curState = sd.analyzeStorage(startOpt, this, true);
// sd is locked but not opened
switch (curState) {
case NORMAL:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf552aa8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
index b1ea5ae..c76fa2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -444,6 +445,41 @@ public class TestDataNodeVolumeFailureReporting {
checkFailuresAtNameNode(dm, dns.get(1), true);
}
+ @Test
+ public void testAutoFormatEmptyDirectory() throws Exception {
+ // remove the version file
+ File dn1Vol1 = cluster.getStorageDir(0, 0);
+ File current = new File(dn1Vol1, "current");
+ File currentVersion = new File(current, "VERSION");
+ currentVersion.delete();
+ // restart the data node
+ assertTrue(cluster.restartDataNodes(true));
+ // the DN should tolerate one volume failure.
+ cluster.waitActive();
+ ArrayList<DataNode> dns = cluster.getDataNodes();
+ DataNode dn = dns.get(0);
+ assertFalse("DataNode should not reformat if VERSION is missing",
+ currentVersion.exists());
+
+ // Make sure DN's JMX sees the failed volume
+ final String[] expectedFailedVolumes = {dn1Vol1.getAbsolutePath()};
+ DataNodeTestUtils.triggerHeartbeat(dn);
+ FsDatasetSpi<?> fsd = dn.getFSDataset();
+ assertEquals(expectedFailedVolumes.length, fsd.getNumFailedVolumes());
+ assertArrayEquals(expectedFailedVolumes, fsd.getFailedStorageLocations());
+ // there shouldn't be any more volume failures due to I/O failure
+ checkFailuresAtDataNode(dn, 0, false, expectedFailedVolumes);
+
+ // The NN reports one volume failures
+ final DatanodeManager dm = cluster.getNamesystem().getBlockManager().
+ getDatanodeManager();
+ long dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0);
+ DFSTestUtil.waitForDatanodeStatus(dm, 1, 0, 1,
+ (1*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(false, 1);
+ checkFailuresAtNameNode(dm, dns.get(0), false, dn1Vol1.getAbsolutePath());
+ }
+
/**
* Checks the NameNode for correct values of aggregate counters tracking failed
* volumes across all DataNodes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf552aa8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
index c55dbae..405d2e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java
@@ -167,6 +167,32 @@ public class TestDataStorage {
}
@Test
+ public void testMissingVersion() throws IOException,
+ URISyntaxException {
+ final int numLocations = 1;
+ final int numNamespace = 1;
+ List<StorageLocation> locations = createStorageLocations(numLocations);
+
+ StorageLocation firstStorage = locations.get(0);
+ Storage.StorageDirectory sd = new Storage.StorageDirectory(
+ firstStorage.getFile());
+ // the directory is not initialized so VERSION does not exist
+ // create a fake directory under current/
+ File currentDir = new File(sd.getCurrentDir(),
+ "BP-787466439-172.26.24.43-1462305406642");
+ assertTrue("unable to mkdir " + currentDir.getName(), currentDir.mkdirs());
+
+ // Add volumes for multiple namespaces.
+ List<NamespaceInfo> namespaceInfos = createNamespaceInfos(numNamespace);
+ for (NamespaceInfo ni : namespaceInfos) {
+ storage.addStorageLocations(mockDN, ni, locations, START_OPT);
+ }
+
+ // It should not format the directory because VERSION is missing.
+ assertTrue("Storage directory was formatted", currentDir.exists());
+ }
+
+ @Test
public void testRecoverTransitionReadFailure() throws IOException {
final int numLocations = 3;
List<StorageLocation> locations =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-4832. NM side resource value
should get updated if change applied in RM side. Contributed by Junping Du
Posted by ar...@apache.org.
YARN-4832. NM side resource value should get updated if change applied in RM side. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa3bc340
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa3bc340
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa3bc340
Branch: refs/heads/HDFS-7240
Commit: fa3bc3405dc2f8497faab45ba5c4de2caf4c29bc
Parents: 7cd5ae6
Author: Jian He <ji...@apache.org>
Authored: Tue May 17 12:51:08 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Tue May 17 12:52:19 2016 -0700
----------------------------------------------------------------------
.../protocolrecords/NodeHeartbeatResponse.java | 4 +
.../RegisterNodeManagerResponse.java | 6 +-
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 46 ++++++++-
.../pb/RegisterNodeManagerResponsePBImpl.java | 46 ++++++++-
.../yarn_server_common_service_protos.proto | 2 +
.../nodemanager/NodeStatusUpdaterImpl.java | 29 +++++-
.../resourcemanager/ResourceTrackerService.java | 67 ++++++++++---
.../yarn/server/resourcemanager/MockNM.java | 21 ++++-
.../resourcemanager/TestRMAdminService.java | 99 ++++++++++++++++++++
9 files changed, 290 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3bc340/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index bd04a0d..386353a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
@@ -77,6 +78,9 @@ public interface NodeHeartbeatResponse {
boolean getAreNodeLabelsAcceptedByRM();
void setAreNodeLabelsAcceptedByRM(boolean areNodeLabelsAcceptedByRM);
+ Resource getResource();
+ void setResource(Resource resource);
+
List<Container> getContainersToDecrease();
void addAllContainersToDecrease(Collection<Container> containersToDecrease);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3bc340/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
index c8678f6..fb1da4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.api.protocolrecords;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
@@ -45,7 +46,10 @@ public interface RegisterNodeManagerResponse {
void setRMVersion(String version);
String getRMVersion();
-
+
+ Resource getResource();
+ void setResource(Resource resource);
+
boolean getAreNodeLabelsAcceptedByRM();
void setAreNodeLabelsAcceptedByRM(boolean areNodeLabelsAcceptedByRM);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3bc340/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index a259158..3422697 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -31,14 +31,17 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerRequest
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ContainerQueuingLimitProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SignalContainerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
@@ -54,17 +57,17 @@ import org.apache.hadoop.yarn.server.api.records.impl.pb.ContainerQueuingLimitPB
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
-
public class NodeHeartbeatResponsePBImpl extends
ProtoBase<NodeHeartbeatResponseProto> implements NodeHeartbeatResponse {
NodeHeartbeatResponseProto proto = NodeHeartbeatResponseProto.getDefaultInstance();
NodeHeartbeatResponseProto.Builder builder = null;
boolean viaProto = false;
-
+
private List<ContainerId> containersToCleanup = null;
private List<ContainerId> containersToBeRemovedFromNM = null;
private List<ApplicationId> applicationsToCleanup = null;
private Map<ApplicationId, ByteBuffer> systemCredentials = null;
+ private Resource resource = null;
private MasterKey containerTokenMasterKey = null;
private MasterKey nmTokenMasterKey = null;
@@ -80,7 +83,7 @@ public class NodeHeartbeatResponsePBImpl extends
this.proto = proto;
viaProto = true;
}
-
+
public NodeHeartbeatResponseProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
@@ -119,6 +122,9 @@ public class NodeHeartbeatResponsePBImpl extends
if (this.containersToSignal != null) {
addContainersToSignalToProto();
}
+ if (this.resource != null) {
+ builder.setResource(convertToProtoFormat(this.resource));
+ }
}
private void addSystemCredentialsToProto() {
@@ -146,8 +152,8 @@ public class NodeHeartbeatResponsePBImpl extends
}
viaProto = false;
}
-
-
+
+
@Override
public int getResponseId() {
NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
@@ -161,6 +167,28 @@ public class NodeHeartbeatResponsePBImpl extends
}
@Override
+ public Resource getResource() {
+ NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.resource != null) {
+ return this.resource;
+ }
+ if (!p.hasResource()) {
+ return null;
+ }
+ this.resource = convertFromProtoFormat(p.getResource());
+ return this.resource;
+ }
+
+ @Override
+ public void setResource(Resource resource) {
+ maybeInitBuilder();
+ if (resource == null) {
+ builder.clearResource();
+ }
+ this.resource = resource;
+ }
+
+ @Override
public MasterKey getContainerTokenMasterKey() {
NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
if (this.containerTokenMasterKey != null) {
@@ -565,6 +593,14 @@ public class NodeHeartbeatResponsePBImpl extends
return ((ContainerIdPBImpl) t).getProto();
}
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource t) {
+ return ((ResourcePBImpl)t).getProto();
+ }
+
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
return new ApplicationIdPBImpl(p);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3bc340/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
index 391d00d..56b675b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
@@ -19,7 +19,10 @@
package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerResponseProto;
@@ -30,17 +33,17 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl;
-
public class RegisterNodeManagerResponsePBImpl extends ProtoBase<RegisterNodeManagerResponseProto> implements RegisterNodeManagerResponse {
RegisterNodeManagerResponseProto proto = RegisterNodeManagerResponseProto.getDefaultInstance();
RegisterNodeManagerResponseProto.Builder builder = null;
boolean viaProto = false;
-
+ private Resource resource = null;
+
private MasterKey containerTokenMasterKey = null;
private MasterKey nmTokenMasterKey = null;
-
+
private boolean rebuild = false;
-
+
public RegisterNodeManagerResponsePBImpl() {
builder = RegisterNodeManagerResponseProto.newBuilder();
}
@@ -49,7 +52,7 @@ public class RegisterNodeManagerResponsePBImpl extends ProtoBase<RegisterNodeMan
this.proto = proto;
viaProto = true;
}
-
+
public RegisterNodeManagerResponseProto getProto() {
if (rebuild)
mergeLocalToProto();
@@ -67,6 +70,9 @@ public class RegisterNodeManagerResponsePBImpl extends ProtoBase<RegisterNodeMan
builder.setNmTokenMasterKey(
convertToProtoFormat(this.nmTokenMasterKey));
}
+ if (this.resource != null) {
+ builder.setResource(convertToProtoFormat(this.resource));
+ }
}
private void mergeLocalToProto() {
@@ -86,6 +92,28 @@ public class RegisterNodeManagerResponsePBImpl extends ProtoBase<RegisterNodeMan
}
@Override
+ public Resource getResource() {
+ RegisterNodeManagerResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.resource != null) {
+ return this.resource;
+ }
+ if (!p.hasResource()) {
+ return null;
+ }
+ this.resource = convertFromProtoFormat(p.getResource());
+ return this.resource;
+ }
+
+ @Override
+ public void setResource(Resource resource) {
+ maybeInitBuilder();
+ if (resource == null) {
+ builder.clearResource();
+ }
+ this.resource = resource;
+ }
+
+ @Override
public MasterKey getContainerTokenMasterKey() {
RegisterNodeManagerResponseProtoOrBuilder p = viaProto ? proto : builder;
if (this.containerTokenMasterKey != null) {
@@ -217,6 +245,14 @@ public class RegisterNodeManagerResponsePBImpl extends ProtoBase<RegisterNodeMan
return ((MasterKeyPBImpl)t).getProto();
}
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource t) {
+ return ((ResourcePBImpl)t).getProto();
+ }
+
@Override
public boolean getAreNodeLabelsAcceptedByRM() {
RegisterNodeManagerResponseProtoOrBuilder p =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3bc340/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index a977653..7d39d30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -63,6 +63,7 @@ message RegisterNodeManagerResponseProto {
optional string diagnostics_message = 5;
optional string rm_version = 6;
optional bool areNodeLabelsAcceptedByRM = 7 [default = false];
+ optional ResourceProto resource = 8;
}
message UnRegisterNodeManagerRequestProto {
@@ -101,6 +102,7 @@ message NodeHeartbeatResponseProto {
repeated ContainerProto containers_to_decrease = 12;
repeated SignalContainerRequestProto containers_to_signal = 13;
optional ContainerQueuingLimitProto container_queuing_limit = 14;
+ optional ResourceProto resource = 15;
}
message ContainerQueuingLimitProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3bc340/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index e02aa0a..f53e6c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
import com.google.common.annotations.VisibleForTesting;
@@ -397,8 +398,17 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
StringBuilder successfullRegistrationMsg = new StringBuilder();
successfullRegistrationMsg.append("Registered with ResourceManager as ")
- .append(this.nodeId).append(" with total resource of ")
- .append(this.totalResource);
+ .append(this.nodeId);
+
+ Resource newResource = regNMResponse.getResource();
+ if (newResource != null) {
+ updateNMResource(newResource);
+ successfullRegistrationMsg.append(" with updated total resource of ")
+ .append(this.totalResource);
+ } else {
+ successfullRegistrationMsg.append(" with total resource of ")
+ .append(this.totalResource);
+ }
successfullRegistrationMsg.append(nodeLabelsHandler
.verifyRMRegistrationResponseForNodeLabels(regNMResponse));
@@ -498,6 +508,12 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
return increasedContainers;
}
+ // Update NM's Resource.
+ private void updateNMResource(Resource resource) {
+ metrics.addResource(Resources.subtract(resource, totalResource));
+ this.totalResource = resource;
+ }
+
// Iterate through the NMContext and clone and get all the containers'
// statuses. If it's a completed container, add into the
// recentlyStoppedContainers collections.
@@ -880,6 +896,15 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
context.getContainerManager().updateQueuingLimit(queuingLimit);
}
}
+ // Handling node resource update case.
+ Resource newResource = response.getResource();
+ if (newResource != null) {
+ updateNMResource(newResource);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Node's resource is updated to " +
+ newResource.toString());
+ }
+ }
} catch (ConnectException e) {
//catch and throw the exception if tried MAX wait time to connect RM
dispatcher.getEventHandler().handle(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3bc340/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index d306b60..f8c9258 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -26,6 +26,9 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -97,6 +100,9 @@ public class ResourceTrackerService extends AbstractService implements
private final RMContainerTokenSecretManager containerTokenSecretManager;
private final NMTokenSecretManagerInRM nmTokenSecretManager;
+ private final ReadLock readLock;
+ private final WriteLock writeLock;
+
private long nextHeartBeatInterval;
private Server server;
private InetSocketAddress resourceTrackerAddress;
@@ -107,7 +113,7 @@ public class ResourceTrackerService extends AbstractService implements
private boolean isDistributedNodeLabelsConf;
private boolean isDelegatedCentralizedNodeLabelsConf;
- private volatile DynamicResourceConfiguration drConf;
+ private DynamicResourceConfiguration drConf;
public ResourceTrackerService(RMContext rmContext,
NodesListManager nodesListManager,
@@ -120,7 +126,9 @@ public class ResourceTrackerService extends AbstractService implements
this.nmLivelinessMonitor = nmLivelinessMonitor;
this.containerTokenSecretManager = containerTokenSecretManager;
this.nmTokenSecretManager = nmTokenSecretManager;
-
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ this.readLock = lock.readLock();
+ this.writeLock = lock.writeLock();
}
@Override
@@ -160,7 +168,6 @@ public class ResourceTrackerService extends AbstractService implements
}
loadDynamicResourceConfiguration(conf);
-
super.serviceInit(conf);
}
@@ -176,6 +183,9 @@ public class ResourceTrackerService extends AbstractService implements
InputStream drInputStream = this.rmContext.getConfigurationProvider()
.getConfigurationInputStream(conf,
YarnConfiguration.DR_CONFIGURATION_FILE);
+ // write lock here on drConfig is unnecessary as here get called at
+ // ResourceTrackerService get initiated and other read and write
+ // operations haven't started yet.
if (drInputStream != null) {
this.drConf = new DynamicResourceConfiguration(conf, drInputStream);
} else {
@@ -192,7 +202,12 @@ public class ResourceTrackerService extends AbstractService implements
*/
public void updateDynamicResourceConfiguration(
DynamicResourceConfiguration conf) {
- this.drConf = conf;
+ this.writeLock.lock();
+ try {
+ this.drConf = conf;
+ } finally {
+ this.writeLock.unlock();
+ }
}
@Override
@@ -233,6 +248,7 @@ public class ResourceTrackerService extends AbstractService implements
if (this.server != null) {
this.server.stop();
}
+
super.serviceStop();
}
@@ -331,16 +347,18 @@ public class ResourceTrackerService extends AbstractService implements
}
// check if node's capacity is load from dynamic-resources.xml
- String[] nodes = this.drConf.getNodes();
String nid = nodeId.toString();
- if (nodes != null && Arrays.asList(nodes).contains(nid)) {
- capability.setMemory(this.drConf.getMemoryPerNode(nid));
- capability.setVirtualCores(this.drConf.getVcoresPerNode(nid));
+ Resource dynamicLoadCapability = loadNodeResourceFromDRConfiguration(nid);
+ if (dynamicLoadCapability != null) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Resource for node: " + nid + " is adjusted to " +
- capability + " due to settings in dynamic-resources.xml.");
+ LOG.debug("Resource for node: " + nid + " is adjusted from: " +
+ capability + " to: " + dynamicLoadCapability +
+ " due to settings in dynamic-resources.xml.");
}
+ capability = dynamicLoadCapability;
+ // sync back with new resource.
+ response.setResource(capability);
}
// Check if this node has minimum allocations
@@ -536,8 +554,17 @@ public class ResourceTrackerService extends AbstractService implements
}
}
- // 6. Send Container Queuing Limits back to the Node. This will be used by
- // the node to truncate the number of Containers queued for execution.
+ // 6. check if node's capacity is load from dynamic-resources.xml
+ // if so, send updated resource back to NM.
+ String nid = nodeId.toString();
+ Resource capability = loadNodeResourceFromDRConfiguration(nid);
+ // sync back with new resource if not null.
+ if (capability != null) {
+ nodeHeartBeatResponse.setResource(capability);
+ }
+
+ // 7. Send Container Queuing Limits back to the Node. This will be used by
+ // the node to truncate the number of Containers queued for execution.
if (this.rmContext.getNodeManagerQueueLimitCalculator() != null) {
nodeHeartBeatResponse.setContainerQueuingLimit(
this.rmContext.getNodeManagerQueueLimitCalculator()
@@ -629,6 +656,22 @@ public class ResourceTrackerService extends AbstractService implements
}
}
+ private Resource loadNodeResourceFromDRConfiguration(String nodeId) {
+ // check if node's capacity is loaded from dynamic-resources.xml
+ this.readLock.lock();
+ try {
+ String[] nodes = this.drConf.getNodes();
+ if (nodes != null && Arrays.asList(nodes).contains(nodeId)) {
+ return Resource.newInstance(this.drConf.getMemoryPerNode(nodeId),
+ this.drConf.getVcoresPerNode(nodeId));
+ } else {
+ return null;
+ }
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
/**
* resolving the network topology.
* @param hostName the hostname of this node.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3bc340/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index 4407fe9..4cec29e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -50,8 +50,8 @@ public class MockNM {
private int responseId;
private NodeId nodeId;
- private final int memory;
- private final int vCores;
+ private int memory;
+ private int vCores;
private ResourceTrackerService resourceTracker;
private int httpPort = 2;
private MasterKey currentContainerTokenMasterKey;
@@ -142,9 +142,14 @@ public class MockNM {
this.currentContainerTokenMasterKey =
registrationResponse.getContainerTokenMasterKey();
this.currentNMTokenMasterKey = registrationResponse.getNMTokenMasterKey();
- return registrationResponse;
+ Resource newResource = registrationResponse.getResource();
+ if (newResource != null) {
+ memory = newResource.getMemory();
+ vCores = newResource.getVirtualCores();
+ }
+ return registrationResponse;
}
-
+
public NodeHeartbeatResponse nodeHeartbeat(boolean isHealthy) throws Exception {
return nodeHeartbeat(new HashMap<ApplicationId, List<ContainerStatus>>(),
isHealthy, ++responseId);
@@ -211,7 +216,13 @@ public class MockNM {
.getKeyId()) {
this.currentNMTokenMasterKey = masterKeyFromRM;
}
-
+
+ Resource newResource = heartbeatResponse.getResource();
+ if (newResource != null) {
+ memory = newResource.getMemory();
+ vCores = newResource.getVirtualCores();
+ }
+
return heartbeatResponse;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa3bc340/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 5c69411..b109639 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -239,6 +239,105 @@ public class TestRMAdminService {
}
@Test
+ public void testRefreshNodesResourceWithResourceReturnInRegistration()
+ throws IOException, YarnException {
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+
+ //upload default configurations
+ uploadDefaultConfiguration();
+
+ MockNM nm = null;
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ nm = rm.registerNode("h1:1234", 2048, 2);
+ } catch(Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ NodeId nid = ConverterUtils.toNodeId("h1:1234");
+ RMNode ni = rm.getRMContext().getRMNodes().get(nid);
+ Resource resource = ni.getTotalCapability();
+ Assert.assertEquals("<memory:2048, vCores:2>", resource.toString());
+
+ DynamicResourceConfiguration drConf =
+ new DynamicResourceConfiguration();
+ drConf.set("yarn.resource.dynamic.nodes", "h1:1234");
+ drConf.set("yarn.resource.dynamic.h1:1234.vcores", "4");
+ drConf.set("yarn.resource.dynamic.h1:1234.memory", "4096");
+ uploadConfiguration(drConf, "dynamic-resources.xml");
+
+ rm.adminService.refreshNodesResources(
+ RefreshNodesResourcesRequest.newInstance());
+
+ try {
+ // register the same node again with original resource.
+ // validate this will get new resource back;
+ nm.registerNode();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ RMNode niAfter = rm.getRMContext().getRMNodes().get(nid);
+ Resource resourceAfter = niAfter.getTotalCapability();
+ Assert.assertEquals("<memory:4096, vCores:4>", resourceAfter.toString());
+
+ Assert.assertEquals(4096, nm.getMemory());
+ Assert.assertEquals(4, nm.getvCores());
+ }
+
+ @Test
+ public void testRefreshNodesResourceWithResourceReturnInHeartbeat()
+ throws IOException, YarnException {
+ configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+
+ //upload default configurations
+ uploadDefaultConfiguration();
+
+ MockNM nm = null;
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ nm = rm.registerNode("h1:1234", 2048, 2);
+ } catch(Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ NodeId nid = ConverterUtils.toNodeId("h1:1234");
+ RMNode ni = rm.getRMContext().getRMNodes().get(nid);
+ Resource resource = ni.getTotalCapability();
+ Assert.assertEquals("<memory:2048, vCores:2>", resource.toString());
+
+ DynamicResourceConfiguration drConf =
+ new DynamicResourceConfiguration();
+ drConf.set("yarn.resource.dynamic.nodes", "h1:1234");
+ drConf.set("yarn.resource.dynamic.h1:1234.vcores", "4");
+ drConf.set("yarn.resource.dynamic.h1:1234.memory", "4096");
+ uploadConfiguration(drConf, "dynamic-resources.xml");
+
+ rm.adminService.refreshNodesResources(
+ RefreshNodesResourcesRequest.newInstance());
+
+ try {
+ // NM-RM heartbeat, validate that this will get new resource back.
+ nm.nodeHeartbeat(true);
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ RMNode niAfter = rm.getRMContext().getRMNodes().get(nid);
+ Resource resourceAfter = niAfter.getTotalCapability();
+ Assert.assertEquals("<memory:4096, vCores:4>", resourceAfter.toString());
+
+ Assert.assertEquals(4096, nm.getMemory());
+ Assert.assertEquals(4, nm.getvCores());
+ }
+
+ @Test
public void testResourcePersistentForNMRegistrationWithNewResource()
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-5075. Fix findbugs warnings
in hadoop-yarn-common module. (asuresh)
Posted by ar...@apache.org.
YARN-5075. Fix findbugs warnings in hadoop-yarn-common module. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ccc93e78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ccc93e78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ccc93e78
Branch: refs/heads/HDFS-7240
Commit: ccc93e78127c14bfd84179395b055c4061ea436a
Parents: 2c91fd8
Author: Arun Suresh <as...@apache.org>
Authored: Mon May 16 23:22:01 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Mon May 16 23:22:01 2016 -0700
----------------------------------------------------------------------
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 2 +-
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +-
.../server/api/records/impl/pb/NodeStatusPBImpl.java | 14 ++++++++------
.../records/impl/pb/QueuedContainersStatusPBImpl.java | 6 +++++-
.../yarn/server/resourcemanager/rmnode/RMNode.java | 2 +-
5 files changed, 16 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccc93e78/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index c640d9f..81c7e6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -327,7 +327,7 @@
<Bug pattern="DM_EXIT" />
</Match>
<Match>
- <Class name="org.apache.hadoop.yarn.server.resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessor" />
+ <Class name="org.apache.hadoop.yarn.event.EventDispatcher$EventProcessor" />
<Method name="run" />
<Bug pattern="DM_EXIT" />
</Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccc93e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 0b150c2..cfe2897 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -339,7 +339,7 @@ public class YarnConfiguration extends Configuration {
600000;
/** K least loaded nodes to be provided to the LocalScheduler of a
- * NodeManager for Distributed Scheduling */
+ * NodeManager for Distributed Scheduling. */
public static final String DIST_SCHEDULING_TOP_K =
YARN_PREFIX + "distributed-scheduling.top-k";
public static final int DIST_SCHEDULING_TOP_K_DEFAULT = 10;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccc93e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
index 9a9a83a..d6a1737 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
@@ -38,10 +38,10 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.QueuedContainersStatusProto;
import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
@@ -404,7 +404,7 @@ public class NodeStatusPBImpl extends NodeStatus {
}
@Override
- public QueuedContainersStatus getQueuedContainersStatus() {
+ public synchronized QueuedContainersStatus getQueuedContainersStatus() {
NodeStatusProtoOrBuilder p =
this.viaProto ? this.proto : this.builder;
if (!p.hasQueuedContainerStatus()) {
@@ -414,7 +414,8 @@ public class NodeStatusPBImpl extends NodeStatus {
}
@Override
- public void setQueuedContainersStatus(QueuedContainersStatus queuedContainersStatus) {
+ public synchronized void setQueuedContainersStatus(
+ QueuedContainersStatus queuedContainersStatus) {
maybeInitBuilder();
if (queuedContainersStatus == null) {
this.builder.clearQueuedContainerStatus();
@@ -457,7 +458,8 @@ public class NodeStatusPBImpl extends NodeStatus {
return ((ApplicationIdPBImpl)c).getProto();
}
- private YarnProtos.ResourceUtilizationProto convertToProtoFormat(ResourceUtilization r) {
+ private YarnProtos.ResourceUtilizationProto convertToProtoFormat(
+ ResourceUtilization r) {
return ((ResourceUtilizationPBImpl) r).getProto();
}
@@ -466,13 +468,13 @@ public class NodeStatusPBImpl extends NodeStatus {
return new ResourceUtilizationPBImpl(p);
}
- private YarnServerCommonProtos.QueuedContainersStatusProto convertToProtoFormat(
+ private QueuedContainersStatusProto convertToProtoFormat(
QueuedContainersStatus r) {
return ((QueuedContainersStatusPBImpl) r).getProto();
}
private QueuedContainersStatus convertFromProtoFormat(
- YarnServerCommonProtos.QueuedContainersStatusProto p) {
+ QueuedContainersStatusProto p) {
return new QueuedContainersStatusPBImpl(p);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccc93e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java
index 54470f4..16b80dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/QueuedContainersStatusPBImpl.java
@@ -21,11 +21,15 @@ package org.apache.hadoop.yarn.server.api.records.impl.pb;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos;
import org.apache.hadoop.yarn.server.api.records.QueuedContainersStatus;
+/**
+ * Protocol Buffer implementation of QueuedContainersStatus.
+ */
public class QueuedContainersStatusPBImpl extends QueuedContainersStatus {
private YarnServerCommonProtos.QueuedContainersStatusProto proto =
YarnServerCommonProtos.QueuedContainersStatusProto.getDefaultInstance();
- private YarnServerCommonProtos.QueuedContainersStatusProto.Builder builder = null;
+ private YarnServerCommonProtos.QueuedContainersStatusProto.Builder builder =
+ null;
private boolean viaProto = false;
public QueuedContainersStatusPBImpl() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccc93e78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 0e281d8..3a9cf54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -170,7 +170,7 @@ public interface RMNode {
public List<Container> pullNewlyIncreasedContainers();
- public QueuedContainersStatus getQueuedContainersStatus();
+ QueuedContainersStatus getQueuedContainersStatus();
long getUntrackedTimeStamp();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-5090. Add End-to-End
test-cases for DistributedScheduling using MiniYarnCluster. (asuresh)
Posted by ar...@apache.org.
YARN-5090. Add End-to-End test-cases for DistributedScheduling using MiniYarnCluster. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a9ecb75
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a9ecb75
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a9ecb75
Branch: refs/heads/HDFS-7240
Commit: 8a9ecb7584b529be1de3a6e8850041e211c88ebc
Parents: 7154ace
Author: Arun Suresh <as...@apache.org>
Authored: Tue May 17 19:01:29 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Tue May 17 19:01:29 2016 -0700
----------------------------------------------------------------------
.../yarn/client/api/impl/TestAMRMProxy.java | 6 +-
.../api/impl/TestDistributedScheduling.java | 307 +++++++++++++++++++
.../TestDistributedSchedulingService.java | 68 ++--
.../hadoop/yarn/server/MiniYARNCluster.java | 5 +
4 files changed, 364 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9ecb75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java
index b92538a..f1e3f03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java
@@ -291,7 +291,7 @@ public class TestAMRMProxy {
}
}
- private ApplicationMasterProtocol createAMRMProtocol(YarnClient rmClient,
+ protected ApplicationMasterProtocol createAMRMProtocol(YarnClient rmClient,
ApplicationId appId, MiniYARNCluster cluster,
final Configuration yarnConf)
throws IOException, InterruptedException, YarnException {
@@ -331,7 +331,7 @@ public class TestAMRMProxy {
});
}
- private AllocateRequest createAllocateRequest(List<NodeReport> listNode) {
+ protected AllocateRequest createAllocateRequest(List<NodeReport> listNode) {
// The test needs AMRMClient to create a real allocate request
AMRMClientImpl<ContainerRequest> amClient =
new AMRMClientImpl<ContainerRequest>();
@@ -361,7 +361,7 @@ public class TestAMRMProxy {
new ArrayList<ContainerId>(), resourceBlacklistRequest);
}
- private ApplicationId createApp(YarnClient yarnClient,
+ protected ApplicationId createApp(YarnClient yarnClient,
MiniYARNCluster yarnCluster) throws Exception {
ApplicationSubmissionContext appContext =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9ecb75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
new file mode 100644
index 0000000..b4dcf66
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
@@ -0,0 +1,307 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api.impl;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Validates End2End Distributed Scheduling flow which includes the AM
+ * specifying OPPORTUNISTIC containers in its resource requests,
+ * the AMRMProxyService on the NM, the LocalScheduler RequestInterceptor on
+ * the NM and the DistributedSchedulingProtocol used by the framework to talk
+ * to the DistributedSchedulingService running on the RM.
+ */
+public class TestDistributedScheduling extends TestAMRMProxy {
+
+ private static final Log LOG =
+ LogFactory.getLog(TestDistributedScheduling.class);
+
+ /**
+ * Validates if Allocate Requests containing only OPPORTUNISTIC container
+ * requests are satisfied instantly.
+ *
+ * @throws Exception
+ */
+ @Test(timeout = 60000)
+ public void testOpportunisticExecutionTypeRequestE2E() throws Exception {
+ MiniYARNCluster cluster =
+ new MiniYARNCluster("testDistributedSchedulingE2E", 1, 1, 1);
+ YarnClient rmClient = null;
+ ApplicationMasterProtocol client;
+
+ try {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_CONTAINER_QUEUING_ENABLED, true);
+ cluster.init(conf);
+ cluster.start();
+ final Configuration yarnConf = cluster.getConfig();
+
+ // the client has to connect to AMRMProxy
+
+ yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
+ rmClient = YarnClient.createYarnClient();
+ rmClient.init(yarnConf);
+ rmClient.start();
+
+ // Submit application
+
+ ApplicationId appId = createApp(rmClient, cluster);
+
+ client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
+
+ LOG.info("testDistributedSchedulingE2E - Register");
+
+ RegisterApplicationMasterResponse responseRegister =
+ client.registerApplicationMaster(RegisterApplicationMasterRequest
+ .newInstance(NetUtils.getHostname(), 1024, ""));
+
+ Assert.assertNotNull(responseRegister);
+ Assert.assertNotNull(responseRegister.getQueue());
+ Assert.assertNotNull(responseRegister.getApplicationACLs());
+ Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
+ Assert
+ .assertNotNull(responseRegister.getContainersFromPreviousAttempts());
+ Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
+ Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
+
+ RMApp rmApp =
+ cluster.getResourceManager().getRMContext().getRMApps().get(appId);
+ Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
+
+ LOG.info("testDistributedSchedulingE2E - Allocate");
+
+ AllocateRequest request =
+ createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
+
+ // Replace 'ANY' requests with OPPORTUNISTIC aks and remove
+ // everything else
+ List<ResourceRequest> newAskList = new ArrayList<>();
+ for (ResourceRequest rr : request.getAskList()) {
+ if (ResourceRequest.ANY.equals(rr.getResourceName())) {
+ ResourceRequest newRR = ResourceRequest.newInstance(rr
+ .getPriority(), rr.getResourceName(),
+ rr.getCapability(), rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), ExecutionType.OPPORTUNISTIC);
+ newAskList.add(newRR);
+ }
+ }
+ request.setAskList(newAskList);
+
+ AllocateResponse allocResponse = client.allocate(request);
+ Assert.assertNotNull(allocResponse);
+
+ // Ensure that all the requests are satisfied immediately
+ Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
+
+ // Verify that the allocated containers are OPPORTUNISTIC
+ for (Container allocatedContainer : allocResponse
+ .getAllocatedContainers()) {
+ ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
+ .newContainerTokenIdentifier(
+ allocatedContainer.getContainerToken());
+ Assert.assertEquals(ExecutionType.OPPORTUNISTIC,
+ containerTokenIdentifier.getExecutionType());
+ }
+
+ LOG.info("testDistributedSchedulingE2E - Finish");
+
+ FinishApplicationMasterResponse responseFinish =
+ client.finishApplicationMaster(FinishApplicationMasterRequest
+ .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
+
+ Assert.assertNotNull(responseFinish);
+
+ } finally {
+ if (rmClient != null) {
+ rmClient.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ /**
+ * Validates if Allocate Requests containing both GUARANTEED and OPPORTUNISTIC
+ * container requests works as expected.
+ *
+ * @throws Exception
+ */
+ @Test(timeout = 60000)
+ public void testMixedExecutionTypeRequestE2E() throws Exception {
+ MiniYARNCluster cluster =
+ new MiniYARNCluster("testDistributedSchedulingE2E", 1, 1, 1);
+ YarnClient rmClient = null;
+ ApplicationMasterProtocol client;
+
+ try {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_CONTAINER_QUEUING_ENABLED, true);
+ cluster.init(conf);
+ cluster.start();
+ final Configuration yarnConf = cluster.getConfig();
+
+ // the client has to connect to AMRMProxy
+
+ yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
+ rmClient = YarnClient.createYarnClient();
+ rmClient.init(yarnConf);
+ rmClient.start();
+
+ // Submit application
+
+ ApplicationId appId = createApp(rmClient, cluster);
+
+ client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
+
+ LOG.info("testDistributedSchedulingE2E - Register");
+
+ RegisterApplicationMasterResponse responseRegister =
+ client.registerApplicationMaster(RegisterApplicationMasterRequest
+ .newInstance(NetUtils.getHostname(), 1024, ""));
+
+ Assert.assertNotNull(responseRegister);
+ Assert.assertNotNull(responseRegister.getQueue());
+ Assert.assertNotNull(responseRegister.getApplicationACLs());
+ Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
+ Assert
+ .assertNotNull(responseRegister.getContainersFromPreviousAttempts());
+ Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
+ Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
+
+ RMApp rmApp =
+ cluster.getResourceManager().getRMContext().getRMApps().get(appId);
+ Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
+
+ LOG.info("testDistributedSchedulingE2E - Allocate");
+
+ AllocateRequest request =
+ createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
+ List<ResourceRequest> askList = request.getAskList();
+ List<ResourceRequest> newAskList = new ArrayList<>(askList);
+
+ // Duplicate all ANY requests marking them as opportunistic
+ for (ResourceRequest rr : askList) {
+ if (ResourceRequest.ANY.equals(rr.getResourceName())) {
+ ResourceRequest newRR = ResourceRequest.newInstance(rr
+ .getPriority(), rr.getResourceName(),
+ rr.getCapability(), rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), ExecutionType.OPPORTUNISTIC);
+ newAskList.add(newRR);
+ }
+ }
+ request.setAskList(newAskList);
+
+ AllocateResponse allocResponse = client.allocate(request);
+ Assert.assertNotNull(allocResponse);
+
+ // Ensure that all the requests are satisfied immediately
+ Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
+
+ // Verify that the allocated containers are OPPORTUNISTIC
+ for (Container allocatedContainer : allocResponse
+ .getAllocatedContainers()) {
+ ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
+ .newContainerTokenIdentifier(
+ allocatedContainer.getContainerToken());
+ Assert.assertEquals(ExecutionType.OPPORTUNISTIC,
+ containerTokenIdentifier.getExecutionType());
+ }
+
+ request.setAskList(new ArrayList<ResourceRequest>());
+ request.setResponseId(request.getResponseId() + 1);
+
+ Thread.sleep(1000);
+
+ // RM should allocate GUARANTEED containers within 2 calls to allocate()
+ allocResponse = client.allocate(request);
+ Assert.assertNotNull(allocResponse);
+ Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
+
+ // Verify that the allocated containers are GUARANTEED
+ for (Container allocatedContainer : allocResponse
+ .getAllocatedContainers()) {
+ ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
+ .newContainerTokenIdentifier(
+ allocatedContainer.getContainerToken());
+ Assert.assertEquals(ExecutionType.GUARANTEED,
+ containerTokenIdentifier.getExecutionType());
+ }
+
+ LOG.info("testDistributedSchedulingE2E - Finish");
+
+ FinishApplicationMasterResponse responseFinish =
+ client.finishApplicationMaster(FinishApplicationMasterRequest
+ .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
+
+ Assert.assertNotNull(responseFinish);
+
+ } finally {
+ if (rmClient != null) {
+ rmClient.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ @Ignore
+ @Override
+ public void testAMRMProxyE2E() throws Exception { }
+
+ @Ignore
+ @Override
+ public void testE2ETokenRenewal() throws Exception { }
+
+ @Ignore
+ @Override
+ public void testE2ETokenSwap() throws Exception { }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9ecb75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java
index 35914c6..49bf4d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDistributedSchedulingService.java
@@ -23,9 +23,18 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB;
-import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb
+ .AllocateResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb
+ .FinishApplicationMasterRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb
+ .FinishApplicationMasterResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb
+ .RegisterApplicationMasterRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb
+ .RegisterApplicationMasterResponsePBImpl;
import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocolPB;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -43,6 +52,10 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
+ .DistSchedAllocateResponsePBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
+ .DistSchedRegisterResponsePBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt
.AMLivelinessMonitor;
@@ -139,18 +152,30 @@ public class TestDistributedSchedulingService {
// ApplicationMasterProtocol clients
RPC.setProtocolEngine(conf, ApplicationMasterProtocolPB.class,
ProtobufRpcEngine.class);
- ApplicationMasterProtocol ampProxy =
- (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol
- .class, NetUtils.getConnectAddress(server), conf);
- RegisterApplicationMasterResponse regResp = ampProxy.registerApplicationMaster(
- factory.newRecordInstance(RegisterApplicationMasterRequest.class));
+ ApplicationMasterProtocolPB ampProxy =
+ RPC.getProxy(ApplicationMasterProtocolPB
+ .class, 1, NetUtils.getConnectAddress(server), conf);
+ RegisterApplicationMasterResponse regResp =
+ new RegisterApplicationMasterResponsePBImpl(
+ ampProxy.registerApplicationMaster(null,
+ ((RegisterApplicationMasterRequestPBImpl)factory
+ .newRecordInstance(
+ RegisterApplicationMasterRequest.class)).getProto()));
Assert.assertEquals("dummyQueue", regResp.getQueue());
- FinishApplicationMasterResponse finishResp = ampProxy
- .finishApplicationMaster(factory.newRecordInstance(
- FinishApplicationMasterRequest.class));
+ FinishApplicationMasterResponse finishResp =
+ new FinishApplicationMasterResponsePBImpl(
+ ampProxy.finishApplicationMaster(null,
+ ((FinishApplicationMasterRequestPBImpl)factory
+ .newRecordInstance(
+ FinishApplicationMasterRequest.class)).getProto()
+ ));
Assert.assertEquals(false, finishResp.getIsUnregistered());
- AllocateResponse allocResp = ampProxy
- .allocate(factory.newRecordInstance(AllocateRequest.class));
+ AllocateResponse allocResp =
+ new AllocateResponsePBImpl(
+ ampProxy.allocate(null,
+ ((AllocateRequestPBImpl)factory
+ .newRecordInstance(AllocateRequest.class)).getProto())
+ );
Assert.assertEquals(12345, allocResp.getNumClusterNodes());
@@ -158,17 +183,22 @@ public class TestDistributedSchedulingService {
// DistributedSchedulerProtocol clients as well
RPC.setProtocolEngine(conf, DistributedSchedulerProtocolPB.class,
ProtobufRpcEngine.class);
- DistributedSchedulerProtocol dsProxy =
- (DistributedSchedulerProtocol) rpc.getProxy(DistributedSchedulerProtocol
- .class, NetUtils.getConnectAddress(server), conf);
+ DistributedSchedulerProtocolPB dsProxy =
+ RPC.getProxy(DistributedSchedulerProtocolPB
+ .class, 1, NetUtils.getConnectAddress(server), conf);
DistSchedRegisterResponse dsRegResp =
- dsProxy.registerApplicationMasterForDistributedScheduling(
- factory.newRecordInstance(RegisterApplicationMasterRequest.class));
+ new DistSchedRegisterResponsePBImpl(
+ dsProxy.registerApplicationMasterForDistributedScheduling(null,
+ ((RegisterApplicationMasterRequestPBImpl)factory
+ .newRecordInstance(RegisterApplicationMasterRequest.class))
+ .getProto()));
Assert.assertEquals(54321l, dsRegResp.getContainerIdStart());
DistSchedAllocateResponse dsAllocResp =
- dsProxy.allocateForDistributedScheduling(
- factory.newRecordInstance(AllocateRequest.class));
+ new DistSchedAllocateResponsePBImpl(
+ dsProxy.allocateForDistributedScheduling(null,
+ ((AllocateRequestPBImpl)factory
+ .newRecordInstance(AllocateRequest.class)).getProto()));
Assert.assertEquals(
"h1", dsAllocResp.getNodesForScheduling().get(0).getHost());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9ecb75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 2704129..2372ea2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -861,6 +861,11 @@ public class MiniYARNCluster extends CompositeService {
localToken);
RequestInterceptor rt = getPipelines()
.get(applicationAttemptId.getApplicationId()).getRootInterceptor();
+ // The DefaultRequestInterceptor will generally be the last
+ // interceptor
+ while (rt.getNextInterceptor() != null) {
+ rt = rt.getNextInterceptor();
+ }
if (rt instanceof DefaultRequestInterceptor) {
((DefaultRequestInterceptor) rt)
.setRMClient(getResourceManager().getApplicationMasterService());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDFS-10397. Distcp should ignore
-delete option if -diff option is provided instead of exiting. Contributed by
Mingliang Liu.
Posted by ar...@apache.org.
HDFS-10397. Distcp should ignore -delete option if -diff option is provided instead of exiting. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03788d30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03788d30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03788d30
Branch: refs/heads/HDFS-7240
Commit: 03788d3015c962eac1a35fa5df39356e8b84731c
Parents: 9478484
Author: Jing Zhao <ji...@apache.org>
Authored: Tue May 17 15:44:07 2016 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Tue May 17 15:46:30 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/tools/DistCpOptions.java | 32 ++++---------
.../org/apache/hadoop/tools/OptionsParser.java | 49 +++++++++-----------
.../apache/hadoop/tools/TestOptionsParser.java | 19 +++++++-
3 files changed, 49 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03788d30/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index 0b1234b..1bc65e0 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -160,7 +160,6 @@ public class DistCpOptions {
* @param atomicCommit - boolean switch
*/
public void setAtomicCommit(boolean atomicCommit) {
- validate(DistCpOptionSwitch.ATOMIC_COMMIT, atomicCommit);
this.atomicCommit = atomicCommit;
}
@@ -179,7 +178,6 @@ public class DistCpOptions {
* @param syncFolder - boolean switch
*/
public void setSyncFolder(boolean syncFolder) {
- validate(DistCpOptionSwitch.SYNC_FOLDERS, syncFolder);
this.syncFolder = syncFolder;
}
@@ -198,7 +196,6 @@ public class DistCpOptions {
* @param deleteMissing - boolean switch
*/
public void setDeleteMissing(boolean deleteMissing) {
- validate(DistCpOptionSwitch.DELETE_MISSING, deleteMissing);
this.deleteMissing = deleteMissing;
}
@@ -254,7 +251,6 @@ public class DistCpOptions {
* @param overwrite - boolean switch
*/
public void setOverwrite(boolean overwrite) {
- validate(DistCpOptionSwitch.OVERWRITE, overwrite);
this.overwrite = overwrite;
}
@@ -270,7 +266,6 @@ public class DistCpOptions {
* update option and CRC is not skipped.
*/
public void setAppend(boolean append) {
- validate(DistCpOptionSwitch.APPEND, append);
this.append = append;
}
@@ -287,7 +282,6 @@ public class DistCpOptions {
}
public void setUseDiff(boolean useDiff, String fromSnapshot, String toSnapshot) {
- validate(DistCpOptionSwitch.DIFF, useDiff);
this.useDiff = useDiff;
this.fromSnapshot = fromSnapshot;
this.toSnapshot = toSnapshot;
@@ -314,7 +308,6 @@ public class DistCpOptions {
* @param skipCRC - boolean switch
*/
public void setSkipCRC(boolean skipCRC) {
- validate(DistCpOptionSwitch.SKIP_CRC, skipCRC);
this.skipCRC = skipCRC;
}
@@ -551,20 +544,15 @@ public class DistCpOptions {
this.filtersFile = filtersFilename;
}
- public void validate(DistCpOptionSwitch option, boolean value) {
-
- boolean syncFolder = (option == DistCpOptionSwitch.SYNC_FOLDERS ?
- value : this.syncFolder);
- boolean overwrite = (option == DistCpOptionSwitch.OVERWRITE ?
- value : this.overwrite);
- boolean deleteMissing = (option == DistCpOptionSwitch.DELETE_MISSING ?
- value : this.deleteMissing);
- boolean atomicCommit = (option == DistCpOptionSwitch.ATOMIC_COMMIT ?
- value : this.atomicCommit);
- boolean skipCRC = (option == DistCpOptionSwitch.SKIP_CRC ?
- value : this.skipCRC);
- boolean append = (option == DistCpOptionSwitch.APPEND ? value : this.append);
- boolean useDiff = (option == DistCpOptionSwitch.DIFF ? value : this.useDiff);
+ void validate() {
+ if (useDiff && deleteMissing) {
+ // -delete and -diff are mutually exclusive. For backward compatibility,
+ // we ignore the -delete option here, instead of throwing an
+ // IllegalArgumentException. See HDFS-10397 for more discussion.
+ OptionsParser.LOG.warn("-delete and -diff are mutually exclusive. " +
+ "The -delete option will be ignored.");
+ setDeleteMissing(false);
+ }
if (syncFolder && atomicCommit) {
throw new IllegalArgumentException("Atomic commit can't be used with " +
@@ -593,7 +581,7 @@ public class DistCpOptions {
throw new IllegalArgumentException(
"Append is disallowed when skipping CRC");
}
- if ((!syncFolder || deleteMissing) && useDiff) {
+ if (!syncFolder && useDiff) {
throw new IllegalArgumentException(
"Diff is valid only with update options");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03788d30/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index 0f0ef2e..5eaf4da 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -41,7 +41,7 @@ import com.google.common.base.Preconditions;
*/
public class OptionsParser {
- private static final Log LOG = LogFactory.getLog(OptionsParser.class);
+ static final Log LOG = LogFactory.getLog(OptionsParser.class);
private static final Options cliOptions = new Options();
@@ -88,14 +88,26 @@ public class OptionsParser {
DistCpOptions option = parseSourceAndTargetPaths(command);
- //Process all the other option switches and set options appropriately
- if (command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch())) {
- option.setIgnoreFailures(true);
- }
+ option.setIgnoreFailures(
+ command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch()));
- if (command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch())) {
- option.setAtomicCommit(true);
- }
+ option.setAtomicCommit(
+ command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch()));
+
+ option.setSyncFolder(
+ command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch()));
+
+ option.setOverwrite(
+ command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch()));
+
+ option.setAppend(
+ command.hasOption(DistCpOptionSwitch.APPEND.getSwitch()));
+
+ option.setDeleteMissing(
+ command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch()));
+
+ option.setSkipCRC(
+ command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch()));
if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch()) &&
option.shouldAtomicCommit()) {
@@ -111,25 +123,6 @@ public class OptionsParser {
option.setLogPath(new Path(getVal(command, DistCpOptionSwitch.LOG_PATH.getSwitch())));
}
- if (command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch())) {
- option.setSyncFolder(true);
- }
-
- if (command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch())) {
- option.setOverwrite(true);
- }
-
- if (command.hasOption(DistCpOptionSwitch.APPEND.getSwitch())) {
- option.setAppend(true);
- }
-
- if (command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch())) {
- option.setDeleteMissing(true);
- }
-
- if (command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch())) {
- option.setSkipCRC(true);
- }
if (command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch())) {
option.setBlocking(false);
@@ -164,6 +157,8 @@ public class OptionsParser {
DistCpOptionSwitch.FILTERS.getSwitch()));
}
+ option.validate();
+
return option;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03788d30/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index b0626ba..35109cc 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.tools;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import org.junit.Assert;
@@ -691,11 +692,25 @@ public class TestOptionsParser {
}
try {
- OptionsParser.parse(new String[] { "-diff", "s1", "s2", "-update", "-delete",
+ options = OptionsParser.parse(new String[] {
+ "-diff", "s1", "s2", "-update", "-delete",
"hdfs://localhost:9820/source/first",
"hdfs://localhost:9820/target/" });
- fail("-diff should fail if -delete option is specified");
+ assertFalse("-delete should be ignored when -diff is specified",
+ options.shouldDeleteMissing());
} catch (IllegalArgumentException e) {
+ fail("Got unexpected IllegalArgumentException: " + e.getMessage());
+ }
+
+ try {
+ options = OptionsParser.parse(new String[] {
+ "-diff", "s1", "s2", "-delete",
+ "hdfs://localhost:9820/source/first",
+ "hdfs://localhost:9820/target/" });
+ fail("-diff should fail if -update option is not specified");
+ } catch (IllegalArgumentException e) {
+ assertFalse("-delete should be ignored when -diff is specified",
+ options.shouldDeleteMissing());
GenericTestUtils.assertExceptionContains(
"Diff is valid only with update options", e);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: Merge branch 'trunk' of
https://git-wip-us.apache.org/repos/asf/hadoop into trunk
Posted by ar...@apache.org.
Merge branch 'trunk' of https://git-wip-us.apache.org/repos/asf/hadoop into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2120de58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2120de58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2120de58
Branch: refs/heads/HDFS-7240
Commit: 2120de588b92b9f22b1cc4188761d6a8c61aa778
Parents: 182fc19 22ff9e6
Author: Kai Zheng <ka...@intel.com>
Authored: Thu May 19 07:17:08 2016 -0700
Committer: Kai Zheng <ka...@intel.com>
Committed: Thu May 19 07:17:08 2016 -0700
----------------------------------------------------------------------
.../web/DelegationTokenAuthenticationFilter.java | 7 +++++--
.../security/token/delegation/web/ServletUtils.java | 7 +++++--
hadoop-project/pom.xml | 5 +++--
.../hadoop/yarn/server/webproxy/WebAppProxyServlet.java | 11 +++++++----
4 files changed, 20 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-4925. ContainerRequest in
AMRMClient,
application should be able to specify nodes/racks together with
nodeLabelExpression. Contributed by Bibin A Chundatt
Posted by ar...@apache.org.
YARN-4925. ContainerRequest in AMRMClient, application should be able to specify nodes/racks together with nodeLabelExpression. Contributed by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f04c81c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f04c81c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f04c81c9
Branch: refs/heads/HDFS-7240
Commit: f04c81c9ce93512bc714531a7731debbe6b794ce
Parents: ef17577
Author: Naganarasimha <na...@apache.org>
Authored: Thu May 19 01:30:12 2016 +0800
Committer: Naganarasimha <na...@apache.org>
Committed: Thu May 19 01:30:48 2016 +0800
----------------------------------------------------------------------
.../hadoop/yarn/client/api/impl/AMRMClientImpl.java | 10 ----------
.../hadoop/yarn/client/api/impl/TestAMRMClient.java | 13 ++++++++++++-
2 files changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f04c81c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index 37a1fb6..46ddc4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -712,16 +712,6 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
"Cannot specify more than two node labels"
+ " in a single node label expression");
}
-
- // Don't allow specify node label against ANY request
- if ((containerRequest.getRacks() != null &&
- (!containerRequest.getRacks().isEmpty()))
- ||
- (containerRequest.getNodes() != null &&
- (!containerRequest.getNodes().isEmpty()))) {
- throw new InvalidContainerRequestException(
- "Cannot specify node label with rack and node");
- }
}
private void validateContainerResourceChangeRequest(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f04c81c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 3b3c0ed..75b49d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -35,11 +35,11 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
@@ -713,6 +713,17 @@ public class TestAMRMClient {
Assert.assertNull(req.getNodeLabelExpression());
}
}
+ // set container with nodes and racks with labels
+ client.addContainerRequest(new ContainerRequest(
+ Resource.newInstance(1024, 1), new String[] { "rack1" },
+ new String[] { "node1", "node2" }, Priority.UNDEFINED, true, "y"));
+ for (ResourceRequest req : client.ask) {
+ if (ResourceRequest.ANY.equals(req.getResourceName())) {
+ Assert.assertEquals("y", req.getNodeLabelExpression());
+ } else {
+ Assert.assertNull(req.getNodeLabelExpression());
+ }
+ }
}
private void verifyAddRequestFailed(AMRMClient<ContainerRequest> client,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDFS-2173. saveNamespace should
not throw IOE when only one storage directory fails to write VERSION file.
Contributed by Andras Bokor.
Posted by ar...@apache.org.
HDFS-2173. saveNamespace should not throw IOE when only one storage directory fails to write VERSION file. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/010e6ac3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/010e6ac3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/010e6ac3
Branch: refs/heads/HDFS-7240
Commit: 010e6ac328855bad59f138b6aeaec535272f448c
Parents: 992a493
Author: Andrew Wang <wa...@apache.org>
Authored: Wed May 18 14:13:41 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed May 18 14:13:41 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/NNStorage.java | 25 ++++++++++++++-
.../hdfs/server/namenode/TestSaveNamespace.java | 33 ++++++++++++++++----
2 files changed, 51 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/010e6ac3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
index ed36b27..44e0ce3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
@@ -1116,4 +1116,27 @@ public class NNStorage extends Storage implements Closeable,
nameDirSizeMap.clear();
nameDirSizeMap.putAll(nnDirSizeMap);
}
-}
+
+ /**
+ * Write all data storage files.
+ * @throws IOException When all the storage directory fails to write
+ * VERSION file
+ */
+ @Override
+ public void writeAll() throws IOException {
+ this.layoutVersion = getServiceLayoutVersion();
+ for (StorageDirectory sd : storageDirs) {
+ try {
+ writeProperties(sd);
+ } catch (Exception e) {
+ LOG.warn("Error during write properties to the VERSION file to " +
+ sd.toString(), e);
+ reportErrorsOnDirectory(sd);
+ if (storageDirs.isEmpty()) {
+ throw new IOException("All the storage failed while writing " +
+ "properties to VERSION file");
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/010e6ac3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
index a03e900..8191cc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
@@ -111,6 +111,29 @@ public class TestSaveNamespace {
}
}
+ private static class FaultyWriteProperties implements Answer<Void> {
+ private int count = 0;
+ private Fault faultType;
+
+ FaultyWriteProperties(Fault faultType) {
+ this.faultType = faultType;
+ }
+
+ @Override
+ public Void answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ StorageDirectory sd = (StorageDirectory)args[0];
+
+ if (faultType == Fault.WRITE_STORAGE_ALL ||
+ (faultType==Fault.WRITE_STORAGE_ONE && count++==1)) {
+ LOG.info("Injecting fault for sd: " + sd);
+ throw new IOException("Injected fault: writeProperties second time");
+ }
+ LOG.info("Not injecting fault for sd: " + sd);
+ return (Void)invocation.callRealMethod();
+ }
+ }
+
private enum Fault {
SAVE_SECOND_FSIMAGE_RTE,
SAVE_SECOND_FSIMAGE_IOE,
@@ -164,17 +187,15 @@ public class TestSaveNamespace {
break;
case WRITE_STORAGE_ALL:
// The spy throws an exception before writing any VERSION files
- doThrow(new RuntimeException("Injected"))
- .when(spyStorage).writeAll();
+ doAnswer(new FaultyWriteProperties(Fault.WRITE_STORAGE_ALL))
+ .when(spyStorage).writeProperties((StorageDirectory)anyObject());
shouldFail = true;
break;
case WRITE_STORAGE_ONE:
// The spy throws on exception on one particular storage directory
- doAnswer(new FaultySaveImage(true))
+ doAnswer(new FaultyWriteProperties(Fault.WRITE_STORAGE_ONE))
.when(spyStorage).writeProperties((StorageDirectory)anyObject());
- // TODO: unfortunately this fails -- should be improved.
- // See HDFS-2173.
- shouldFail = true;
+ shouldFail = false;
break;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HADOOP-13146. Refactor
RetryInvocationHandler. Contributed by Tsz Wo Nicholas Sze.
Posted by ar...@apache.org.
HADOOP-13146. Refactor RetryInvocationHandler. Contributed by Tsz Wo Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9a8297c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9a8297c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9a8297c
Branch: refs/heads/HDFS-7240
Commit: a9a8297cad4122961b34265c0a31d87134a4a028
Parents: 1217c8f
Author: Jing Zhao <ji...@apache.org>
Authored: Mon May 16 15:23:36 2016 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Mon May 16 15:23:36 2016 -0700
----------------------------------------------------------------------
.../hadoop/io/retry/FailoverProxyProvider.java | 11 +
.../hadoop/io/retry/RetryInvocationHandler.java | 350 ++++++++++---------
.../apache/hadoop/io/retry/TestRetryProxy.java | 67 ++--
3 files changed, 211 insertions(+), 217 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9a8297c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java
index 5acb936..c73e083 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/FailoverProxyProvider.java
@@ -37,10 +37,21 @@ public interface FailoverProxyProvider<T> extends Closeable {
* provides information for debugging purposes.
*/
public final String proxyInfo;
+
public ProxyInfo(T proxy, String proxyInfo) {
this.proxy = proxy;
this.proxyInfo = proxyInfo;
}
+
+ public String getString(String methodName) {
+ return proxy.getClass().getSimpleName() + "." + methodName
+ + " over " + proxyInfo;
+ }
+
+ @Override
+ public String toString() {
+ return proxy.getClass().getSimpleName() + " over " + proxyInfo;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9a8297c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
index a67c84f..300d0c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
@@ -17,48 +17,137 @@
*/
package org.apache.hadoop.io.retry;
-import java.io.IOException;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.retry.FailoverProxyProvider.ProxyInfo;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
-import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.*;
import org.apache.hadoop.ipc.Client.ConnectionId;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RpcConstants;
-import org.apache.hadoop.ipc.RpcInvocationHandler;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
/**
- * This class implements RpcInvocationHandler and supports retry on the client
- * side.
+ * A {@link RpcInvocationHandler} which supports client side retry .
*/
@InterfaceAudience.Private
public class RetryInvocationHandler<T> implements RpcInvocationHandler {
public static final Log LOG = LogFactory.getLog(RetryInvocationHandler.class);
- private final FailoverProxyProvider<T> proxyProvider;
- /**
- * The number of times the associated proxyProvider has ever been failed over.
- */
- private long proxyProviderFailoverCount = 0;
+ private static class Counters {
+ /** Counter for retries. */
+ private int retries;
+ /** Counter for method invocation has been failed over. */
+ private int failovers;
+ }
+
+ private static class ProxyDescriptor<T> {
+ private final FailoverProxyProvider<T> fpp;
+ /** Count the associated proxy provider has ever been failed over. */
+ private long failoverCount = 0;
+
+ private ProxyInfo<T> proxyInfo;
+
+ ProxyDescriptor(FailoverProxyProvider<T> fpp) {
+ this.fpp = fpp;
+ this.proxyInfo = fpp.getProxy();
+ }
+
+ synchronized ProxyInfo<T> getProxyInfo() {
+ return proxyInfo;
+ }
+
+ synchronized T getProxy() {
+ return proxyInfo.proxy;
+ }
+
+ synchronized long getFailoverCount() {
+ return failoverCount;
+ }
+
+ synchronized void failover(long expectedFailoverCount, Method method) {
+ // Make sure that concurrent failed invocations only cause a single
+ // actual failover.
+ if (failoverCount == expectedFailoverCount) {
+ fpp.performFailover(proxyInfo.proxy);
+ failoverCount++;
+ } else {
+ LOG.warn("A failover has occurred since the start of "
+ + proxyInfo.getString(method.getName()));
+ }
+ proxyInfo = fpp.getProxy();
+ }
+
+ boolean idempotentOrAtMostOnce(Method method) throws NoSuchMethodException {
+ final Method m = fpp.getInterface()
+ .getMethod(method.getName(), method.getParameterTypes());
+ return m.isAnnotationPresent(Idempotent.class)
+ || m.isAnnotationPresent(AtMostOnce.class);
+ }
+
+ void close() throws IOException {
+ fpp.close();
+ }
+ }
+
+ private static class RetryInfo {
+ private final long delay;
+ private final RetryAction failover;
+ private final RetryAction fail;
+
+ RetryInfo(long delay, RetryAction failover, RetryAction fail) {
+ this.delay = delay;
+ this.failover = failover;
+ this.fail = fail;
+ }
+
+ static RetryInfo newRetryInfo(RetryPolicy policy, Exception e,
+ Counters counters, boolean idempotentOrAtMostOnce) throws Exception {
+ long maxRetryDelay = 0;
+ RetryAction failover = null;
+ RetryAction retry = null;
+ RetryAction fail = null;
+
+ final Iterable<Exception> exceptions = e instanceof MultiException ?
+ ((MultiException) e).getExceptions().values()
+ : Collections.singletonList(e);
+ for (Exception exception : exceptions) {
+ final RetryAction a = policy.shouldRetry(exception,
+ counters.retries, counters.failovers, idempotentOrAtMostOnce);
+ if (a.action == RetryAction.RetryDecision.FAIL) {
+ fail = a;
+ } else {
+ // must be a retry or failover
+ if (a.action == RetryAction.RetryDecision.FAILOVER_AND_RETRY) {
+ failover = a;
+ } else {
+ retry = a;
+ }
+ if (a.delayMillis > maxRetryDelay) {
+ maxRetryDelay = a.delayMillis;
+ }
+ }
+ }
+
+ return new RetryInfo(maxRetryDelay, failover,
+ failover == null && retry == null? fail: null);
+ }
+ }
+
+ private final ProxyDescriptor<T> proxyDescriptor;
+
private volatile boolean hasMadeASuccessfulCall = false;
private final RetryPolicy defaultPolicy;
private final Map<String,RetryPolicy> methodNameToPolicyMap;
- private ProxyInfo<T> currentProxy;
protected RetryInvocationHandler(FailoverProxyProvider<T> proxyProvider,
RetryPolicy retryPolicy) {
@@ -68,39 +157,40 @@ public class RetryInvocationHandler<T> implements RpcInvocationHandler {
protected RetryInvocationHandler(FailoverProxyProvider<T> proxyProvider,
RetryPolicy defaultPolicy,
Map<String, RetryPolicy> methodNameToPolicyMap) {
- this.proxyProvider = proxyProvider;
+ this.proxyDescriptor = new ProxyDescriptor<>(proxyProvider);
this.defaultPolicy = defaultPolicy;
this.methodNameToPolicyMap = methodNameToPolicyMap;
- this.currentProxy = proxyProvider.getProxy();
+ }
+
+ private RetryPolicy getRetryPolicy(Method method) {
+ final RetryPolicy policy = methodNameToPolicyMap.get(method.getName());
+ return policy != null? policy: defaultPolicy;
}
@Override
public Object invoke(Object proxy, Method method, Object[] args)
- throws Throwable {
- RetryPolicy policy = methodNameToPolicyMap.get(method.getName());
- if (policy == null) {
- policy = defaultPolicy;
- }
-
- // The number of times this method invocation has been failed over.
- int invocationFailoverCount = 0;
- final boolean isRpc = isRpcInvocation(currentProxy.proxy);
+ throws Throwable {
+ final boolean isRpc = isRpcInvocation(proxyDescriptor.getProxy());
final int callId = isRpc? Client.nextCallId(): RpcConstants.INVALID_CALL_ID;
- int retries = 0;
+ return invoke(method, args, isRpc, callId, new Counters());
+ }
+
+ private Object invoke(final Method method, final Object[] args,
+ final boolean isRpc, final int callId, final Counters counters)
+ throws Throwable {
+ final RetryPolicy policy = getRetryPolicy(method);
+
while (true) {
// The number of times this invocation handler has ever been failed over,
// before this method invocation attempt. Used to prevent concurrent
// failed method invocations from triggering multiple failover attempts.
- long invocationAttemptFailoverCount;
- synchronized (proxyProvider) {
- invocationAttemptFailoverCount = proxyProviderFailoverCount;
- }
+ final long failoverCount = proxyDescriptor.getFailoverCount();
if (isRpc) {
- Client.setCallIdAndRetryCount(callId, retries);
+ Client.setCallIdAndRetryCount(callId, counters.retries);
}
try {
- Object ret = invokeMethod(method, args);
+ final Object ret = invokeMethod(method, args);
hasMadeASuccessfulCall = true;
return ret;
} catch (Exception ex) {
@@ -108,153 +198,74 @@ public class RetryInvocationHandler<T> implements RpcInvocationHandler {
// If interrupted, do not retry.
throw ex;
}
- boolean isIdempotentOrAtMostOnce = proxyProvider.getInterface()
- .getMethod(method.getName(), method.getParameterTypes())
- .isAnnotationPresent(Idempotent.class);
- if (!isIdempotentOrAtMostOnce) {
- isIdempotentOrAtMostOnce = proxyProvider.getInterface()
- .getMethod(method.getName(), method.getParameterTypes())
- .isAnnotationPresent(AtMostOnce.class);
- }
- List<RetryAction> actions = extractActions(policy, ex, retries++,
- invocationFailoverCount, isIdempotentOrAtMostOnce);
- RetryAction failAction = getFailAction(actions);
- if (failAction != null) {
- // fail.
- if (failAction.reason != null) {
- LOG.warn("Exception while invoking " + currentProxy.proxy.getClass()
- + "." + method.getName() + " over " + currentProxy.proxyInfo
- + ". Not retrying because " + failAction.reason, ex);
- }
- throw ex;
- } else { // retry or failover
- // avoid logging the failover if this is the first call on this
- // proxy object, and we successfully achieve the failover without
- // any flip-flopping
- boolean worthLogging =
- !(invocationFailoverCount == 0 && !hasMadeASuccessfulCall);
- worthLogging |= LOG.isDebugEnabled();
- RetryAction failOverAction = getFailOverAction(actions);
- long delay = getDelayMillis(actions);
-
- if (worthLogging) {
- String msg = "Exception while invoking " + method.getName()
- + " of class " + currentProxy.proxy.getClass().getSimpleName()
- + " over " + currentProxy.proxyInfo;
-
- if (invocationFailoverCount > 0) {
- msg += " after " + invocationFailoverCount + " fail over attempts";
- }
-
- if (failOverAction != null) {
- // failover
- msg += ". Trying to fail over " + formatSleepMessage(delay);
- } else {
- // retry
- msg += ". Retrying " + formatSleepMessage(delay);
- }
- LOG.info(msg, ex);
- }
-
- if (delay > 0) {
- Thread.sleep(delay);
- }
-
- if (failOverAction != null) {
- // Make sure that concurrent failed method invocations only cause a
- // single actual fail over.
- synchronized (proxyProvider) {
- if (invocationAttemptFailoverCount == proxyProviderFailoverCount) {
- proxyProvider.performFailover(currentProxy.proxy);
- proxyProviderFailoverCount++;
- } else {
- LOG.warn("A failover has occurred since the start of this method"
- + " invocation attempt.");
- }
- currentProxy = proxyProvider.getProxy();
- }
- invocationFailoverCount++;
- }
- }
+ handleException(method, policy, failoverCount, counters, ex);
}
}
}
- /**
- * Obtain a retry delay from list of RetryActions.
- */
- private long getDelayMillis(List<RetryAction> actions) {
- long retVal = 0;
- for (RetryAction action : actions) {
- if (action.action == RetryAction.RetryDecision.FAILOVER_AND_RETRY ||
- action.action == RetryAction.RetryDecision.RETRY) {
- if (action.delayMillis > retVal) {
- retVal = action.delayMillis;
- }
+ private void handleException(final Method method, final RetryPolicy policy,
+ final long expectedFailoverCount, final Counters counters,
+ final Exception ex) throws Exception {
+ final RetryInfo retryInfo = RetryInfo.newRetryInfo(policy, ex, counters,
+ proxyDescriptor.idempotentOrAtMostOnce(method));
+ counters.retries++;
+
+ if (retryInfo.fail != null) {
+ // fail.
+ if (retryInfo.fail.reason != null) {
+ LOG.warn("Exception while invoking "
+ + proxyDescriptor.getProxyInfo().getString(method.getName())
+ + ". Not retrying because " + retryInfo.fail.reason, ex);
}
+ throw ex;
}
- return retVal;
- }
- /**
- * Return the first FAILOVER_AND_RETRY action.
- */
- private RetryAction getFailOverAction(List<RetryAction> actions) {
- for (RetryAction action : actions) {
- if (action.action == RetryAction.RetryDecision.FAILOVER_AND_RETRY) {
- return action;
- }
+ // retry
+ final boolean isFailover = retryInfo.failover != null;
+
+ log(method, isFailover, counters.failovers, retryInfo.delay, ex);
+
+ if (retryInfo.delay > 0) {
+ Thread.sleep(retryInfo.delay);
}
- return null;
- }
- /**
- * Return the last FAIL action.. only if there are no RETRY actions.
- */
- private RetryAction getFailAction(List<RetryAction> actions) {
- RetryAction fAction = null;
- for (RetryAction action : actions) {
- if (action.action == RetryAction.RetryDecision.FAIL) {
- fAction = action;
- } else {
- // Atleast 1 RETRY
- return null;
- }
+ if (isFailover) {
+ proxyDescriptor.failover(expectedFailoverCount, method);
+ counters.failovers++;
}
- return fAction;
}
- private List<RetryAction> extractActions(RetryPolicy policy, Exception ex,
- int i, int invocationFailoverCount,
- boolean isIdempotentOrAtMostOnce)
- throws Exception {
- List<RetryAction> actions = new LinkedList<>();
- if (ex instanceof MultiException) {
- for (Exception th : ((MultiException) ex).getExceptions().values()) {
- actions.add(policy.shouldRetry(th, i, invocationFailoverCount,
- isIdempotentOrAtMostOnce));
- }
- } else {
- actions.add(policy.shouldRetry(ex, i,
- invocationFailoverCount, isIdempotentOrAtMostOnce));
+ private void log(final Method method, final boolean isFailover,
+ final int failovers, final long delay, final Exception ex) {
+ // log info if this has made some successful calls or
+ // this is not the first failover
+ final boolean info = hasMadeASuccessfulCall || failovers != 0;
+ if (!info && !LOG.isDebugEnabled()) {
+ return;
}
- return actions;
- }
- private static String formatSleepMessage(long millis) {
- if (millis > 0) {
- return "after sleeping for " + millis + "ms.";
+ final StringBuilder b = new StringBuilder()
+ .append("Exception while invoking ")
+ .append(proxyDescriptor.getProxyInfo().getString(method.getName()));
+ if (failovers > 0) {
+ b.append(" after ").append(failovers).append(" failover attempts");
+ }
+ b.append(isFailover? ". Trying to failover ": ". Retrying ");
+ b.append(delay > 0? "after sleeping for " + delay + "ms.": "immediately.");
+
+ if (info) {
+ LOG.info(b.toString(), ex);
} else {
- return "immediately.";
+ LOG.debug(b.toString(), ex);
}
}
-
+
protected Object invokeMethod(Method method, Object[] args) throws Throwable {
try {
if (!method.isAccessible()) {
method.setAccessible(true);
}
- return method.invoke(currentProxy.proxy, args);
+ return method.invoke(proxyDescriptor.getProxy(), args);
} catch (InvocationTargetException e) {
throw e.getCause();
}
@@ -274,12 +285,11 @@ public class RetryInvocationHandler<T> implements RpcInvocationHandler {
@Override
public void close() throws IOException {
- proxyProvider.close();
+ proxyDescriptor.close();
}
@Override //RpcInvocationHandler
public ConnectionId getConnectionId() {
- return RPC.getConnectionIdForProxy(currentProxy.proxy);
+ return RPC.getConnectionIdForProxy(proxyDescriptor.getProxy());
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9a8297c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
index 4137dae..41c1be4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
@@ -18,55 +18,32 @@
package org.apache.hadoop.io.retry;
-import static org.apache.hadoop.io.retry.RetryPolicies.RETRY_FOREVER;
-import static org.apache.hadoop.io.retry.RetryPolicies.TRY_ONCE_THEN_FAIL;
-import static org.apache.hadoop.io.retry.RetryPolicies.retryByException;
-import static org.apache.hadoop.io.retry.RetryPolicies.retryByRemoteException;
-import static org.apache.hadoop.io.retry.RetryPolicies.retryOtherThanRemoteException;
-import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithFixedSleep;
-import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithProportionalSleep;
-import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumTimeWithFixedSleep;
-import static org.apache.hadoop.io.retry.RetryPolicies.retryForeverWithFixedSleep;
-import static org.apache.hadoop.io.retry.RetryPolicies.exponentialBackoffRetry;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
+import org.apache.hadoop.io.retry.RetryPolicies.*;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision;
-import org.apache.hadoop.io.retry.RetryPolicies.RetryUpToMaximumCountWithFixedSleep;
-import org.apache.hadoop.io.retry.RetryPolicies.RetryUpToMaximumTimeWithFixedSleep;
-import org.apache.hadoop.io.retry.RetryPolicies.TryOnceThenFail;
import org.apache.hadoop.io.retry.UnreliableInterface.FatalException;
import org.apache.hadoop.io.retry.UnreliableInterface.UnreliableException;
import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ipc.RemoteException;
-
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
import org.junit.Before;
import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.hadoop.io.retry.RetryPolicies.*;
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Mockito.*;
public class TestRetryProxy {
@@ -131,25 +108,21 @@ public class TestRetryProxy {
final UnreliableInterface unreliable = (UnreliableInterface)
RetryProxy.create(UnreliableInterface.class, unreliableImpl, RETRY_FOREVER);
assertTrue(RetryInvocationHandler.isRpcInvocation(unreliable));
-
+
+ final AtomicInteger count = new AtomicInteger();
// Embed the proxy in ProtocolTranslator
ProtocolTranslator xlator = new ProtocolTranslator() {
- int count = 0;
@Override
public Object getUnderlyingProxyObject() {
- count++;
+ count.getAndIncrement();
return unreliable;
}
- @Override
- public String toString() {
- return "" + count;
- }
};
// For a proxy wrapped in ProtocolTranslator method should return true
assertTrue(RetryInvocationHandler.isRpcInvocation(xlator));
// Ensure underlying proxy was looked at
- assertEquals(xlator.toString(), "1");
+ assertEquals(1, count.get());
// For non-proxy the method must return false
assertFalse(RetryInvocationHandler.isRpcInvocation(new Object()));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HDFS-10404. Fix formatting of
CacheAdmin command usage help text (Yiqun Lin via cmccabe)
Posted by ar...@apache.org.
HDFS-10404. Fix formatting of CacheAdmin command usage help text (Yiqun Lin via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7cd5ae62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7cd5ae62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7cd5ae62
Branch: refs/heads/HDFS-7240
Commit: 7cd5ae62f639a857f24f5463f2aefc099c631a14
Parents: 34fddd1
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue May 17 11:10:48 2016 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue May 17 11:10:48 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java | 6 ++++--
.../src/site/markdown/CentralizedCacheManagement.md | 2 +-
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 4 ++--
3 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7cd5ae62/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
index 6888ea8..7912c3a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
@@ -443,7 +443,8 @@ public class CacheAdmin extends Configured implements Tool {
@Override
public String getShortUsage() {
return "[" + getName()
- + " [-stats] [-path <path>] [-pool <pool>] [-id <id>]\n";
+ + " [-stats] [-path <path>] [-pool <pool>] [-id <id>]"
+ + "]\n";
}
@Override
@@ -554,7 +555,8 @@ public class CacheAdmin extends Configured implements Tool {
public String getShortUsage() {
return "[" + NAME + " <name> [-owner <owner>] " +
"[-group <group>] [-mode <mode>] [-limit <limit>] " +
- "[-maxTtl <maxTtl>]\n";
+ "[-maxTtl <maxTtl>]" +
+ "]\n";
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7cd5ae62/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
index 72c125d..210d25c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
@@ -157,7 +157,7 @@ List cache directives.
#### addPool
-Usage: `hdfs cacheadmin -addPool <name> [-owner <owner>] [-group <group>] [-mode <mode>] [-limit <limit>] [-maxTtl <maxTtl`\>
+Usage: `hdfs cacheadmin -addPool <name> [-owner <owner>] [-group <group>] [-mode <mode>] [-limit <limit>] [-maxTtl <maxTtl>]`
Add a new cache pool.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7cd5ae62/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 13c5f0c..7c94e92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -321,10 +321,10 @@ Usage:
hdfs cacheadmin [-addDirective -path <path> -pool <pool-name> [-force] [-replication <replication>] [-ttl <time-to-live>]]
hdfs cacheadmin [-modifyDirective -id <id> [-path <path>] [-force] [-replication <replication>] [-pool <pool-name>] [-ttl <time-to-live>]]
- hdfs cacheadmin [-listDirectives [-stats] [-path <path>] [-pool <pool>] [-id <id>]
+ hdfs cacheadmin [-listDirectives [-stats] [-path <path>] [-pool <pool>] [-id <id>]]
hdfs cacheadmin [-removeDirective <id>]
hdfs cacheadmin [-removeDirectives -path <path>]
- hdfs cacheadmin [-addPool <name> [-owner <owner>] [-group <group>] [-mode <mode>] [-limit <limit>] [-maxTtl <maxTtl>]
+ hdfs cacheadmin [-addPool <name> [-owner <owner>] [-group <group>] [-mode <mode>] [-limit <limit>] [-maxTtl <maxTtl>]]
hdfs cacheadmin [-modifyPool <name> [-owner <owner>] [-group <group>] [-mode <mode>] [-limit <limit>] [-maxTtl <maxTtl>]]
hdfs cacheadmin [-removePool <name>]
hdfs cacheadmin [-listPools [-stats] [<name>]]
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HADOOP-13161. remove JDK7 from
Dockerfile (aw)
Posted by ar...@apache.org.
HADOOP-13161. remove JDK7 from Dockerfile (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c91fd82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c91fd82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c91fd82
Branch: refs/heads/HDFS-7240
Commit: 2c91fd82415a7b0fcdf91952bda98bea669f1646
Parents: 61f46be
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon May 16 20:25:40 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon May 16 20:25:40 2016 -0700
----------------------------------------------------------------------
dev-support/docker/Dockerfile | 26 ++++++++++++++------------
1 file changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c91fd82/dev-support/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index eba7df7..f9bf5aa 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -28,9 +28,11 @@ ENV DEBCONF_TERSE true
######
# Install common dependencies from packages
+#
+# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
+# Ubuntu Java. See Java section below!
######
RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
- ant \
build-essential \
bzip2 \
cmake \
@@ -42,17 +44,14 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
git \
gnupg-agent \
make \
- maven \
libbz2-dev \
libcurl4-openssl-dev \
libfuse-dev \
- libjansson-dev \
libprotobuf-dev \
libprotoc-dev \
libsnappy-dev \
libssl-dev \
libtool \
- openjdk-7-jdk \
pinentry-curses \
pkg-config \
protobuf-compiler \
@@ -63,10 +62,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
snappy \
zlib1g-dev
-# Fixing the Apache commons / Maven dependency problem under Ubuntu:
-# See http://wiki.apache.org/commons/VfsProblems
-RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
-
######
# Install ISA-L library
######
@@ -87,13 +82,20 @@ RUN add-apt-repository -y ppa:webupd8team/java
RUN apt-get -q update
# Auto-accept the Oracle JDK license
-RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-RUN apt-get -q install --no-install-recommends -y oracle-java7-installer
-
-# Auto-accept the Oracle JDK license
RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
RUN apt-get -q install --no-install-recommends -y oracle-java8-installer
+####
+# Apps that require Java
+###
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
+ ant \
+ maven
+
+# Fixing the Apache commons / Maven dependency problem under Ubuntu:
+# See http://wiki.apache.org/commons/VfsProblems
+RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
+
######
# Install findbugs
######
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-4913. Yarn logs should take a
-out option to write to a directory. Contributed by Xuan Gong.
Posted by ar...@apache.org.
YARN-4913. Yarn logs should take a -out option to write to a directory. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef175779
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef175779
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef175779
Branch: refs/heads/HDFS-7240
Commit: ef1757790d89cc72f88f5330761b1c8901c59e94
Parents: bad85f3
Author: Varun Vasudev <vv...@apache.org>
Authored: Wed May 18 22:44:33 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Wed May 18 22:44:33 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 391 +++++++++++--------
.../hadoop/yarn/client/cli/TestLogsCLI.java | 106 ++++-
.../yarn/logaggregation/LogCLIHelpers.java | 128 ++++--
3 files changed, 419 insertions(+), 206 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef175779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 1ac4d61..039ba3c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client.cli;
import java.io.IOException;
+import java.io.PrintStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
@@ -81,15 +82,13 @@ public class LogsCLI extends Configured implements Tool {
private static final String CONTAINER_LOG_FILES = "logFiles";
private static final String SHOW_META_INFO = "show_meta_info";
private static final String LIST_NODES_OPTION = "list_nodes";
+ private static final String OUT_OPTION = "out";
public static final String HELP_CMD = "help";
@Override
public int run(String[] args) throws Exception {
-
Options opts = createCommandOpts();
-
Options printOpts = createPrintOpts(opts);
-
if (args.length < 1) {
printHelpMessage(printOpts);
return -1;
@@ -108,6 +107,7 @@ public class LogsCLI extends Configured implements Tool {
boolean nodesList = false;
String[] logFiles = null;
List<String> amContainersList = new ArrayList<String>();
+ String localDir = null;
try {
CommandLine commandLine = parser.parse(opts, args, true);
appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION);
@@ -117,6 +117,7 @@ public class LogsCLI extends Configured implements Tool {
getAMContainerLogs = commandLine.hasOption(AM_CONTAINER_OPTION);
showMetaInfo = commandLine.hasOption(SHOW_META_INFO);
nodesList = commandLine.hasOption(LIST_NODES_OPTION);
+ localDir = commandLine.getOptionValue(OUT_OPTION);
if (getAMContainerLogs) {
try {
amContainersList = parseAMContainer(commandLine, printOpts);
@@ -151,7 +152,6 @@ public class LogsCLI extends Configured implements Tool {
LogCLIHelpers logCliHelper = new LogCLIHelpers();
logCliHelper.setConf(getConf());
- boolean appStateObtainedSuccessfully = true;
YarnApplicationState appState = YarnApplicationState.NEW;
ApplicationReport appReport = null;
try {
@@ -164,7 +164,9 @@ public class LogsCLI extends Configured implements Tool {
return -1;
}
} catch (IOException | YarnException e) {
- appStateObtainedSuccessfully = false;
+ // If we can not get appReport from either RM or ATS
+ // We will assume that this app has already finished.
+ appState = YarnApplicationState.FINISHED;
System.err.println("Unable to get ApplicationState."
+ " Attempting to fetch logs directly from the filesystem.");
}
@@ -179,19 +181,22 @@ public class LogsCLI extends Configured implements Tool {
}
}
+ ContainerLogsRequest request = new ContainerLogsRequest(appId,
+ isApplicationFinished(appState), appOwner,
+ nodeAddress, null, containerIdStr);
+
if (showMetaInfo) {
- return showMetaInfo(appState, appStateObtainedSuccessfully,
- logCliHelper, appId, containerIdStr, nodeAddress, appOwner);
+ return showMetaInfo(request, logCliHelper);
}
if (nodesList) {
- return showNodeLists(appState, appStateObtainedSuccessfully,
- logCliHelper, appId, appOwner);
+ return showNodeLists(request, logCliHelper);
}
+
// To get am logs
if (getAMContainerLogs) {
- return fetchAMContainerLogs(logFiles, appState, appId, appOwner,
- amContainersList, logCliHelper);
+ return fetchAMContainerLogs(request, amContainersList,
+ logFiles, logCliHelper, localDir);
}
int resultCode = 0;
@@ -203,16 +208,12 @@ public class LogsCLI extends Configured implements Tool {
+ " does not have the container:" + containerId);
return -1;
}
- return fetchContainerLogs(appState, appStateObtainedSuccessfully,
- logFiles, appOwner, nodeAddress, containerId, logCliHelper);
+ return fetchContainerLogs(request, logFiles,
+ logCliHelper, localDir);
} else {
if (nodeAddress == null) {
- resultCode =
- logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out);
- if (resultCode == -1) {
- System.err.println("Can not find the logs for the application: "
- + appId + " with the appOwner: " + appOwner);
- }
+ resultCode = fetchApplicationLogs(appId, appOwner,
+ logCliHelper, localDir);
} else {
System.err.println("Should at least provide ContainerId!");
printHelpMessage(printOpts);
@@ -351,69 +352,74 @@ public class LogsCLI extends Configured implements Tool {
}
private void printContainerLogsFromRunningApplication(Configuration conf,
- ContainerId containerId, String nodeHttpAddress,
- String nodeId, String[] logFiles, LogCLIHelpers logCliHelper,
- String appOwner) throws IOException {
- String appId = containerId.getApplicationAttemptId()
- .getApplicationId().toString();
- String containerIdStr = containerId.toString();
+ ContainerLogsRequest request, String[] logFiles,
+ LogCLIHelpers logCliHelper, String localDir) throws IOException {
+ String appId = request.getAppId().toString();
+ String containerIdStr = request.getContainerId().toString();
String[] requestedLogFiles = logFiles;
- // fetch all the log files for the container
- if (fetchAllLogFiles(logFiles)) {
- requestedLogFiles =
- getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
- }
- Client webServiceClient = Client.create();
- String containerString = "\n\nContainer: " + containerIdStr;
- System.out.println(containerString);
- System.out.println(StringUtils.repeat("=", containerString.length()));
-
- for (String logFile : requestedLogFiles) {
- System.out.println("LogType:" + logFile);
- System.out.println("Log Upload Time:"
- + Times.format(System.currentTimeMillis()));
- System.out.println("Log Contents:");
- try {
- WebResource webResource =
- webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf)
- + nodeHttpAddress);
- ClientResponse response =
- webResource.path("ws").path("v1").path("node")
- .path("containerlogs").path(containerIdStr).path(logFile)
- .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
- System.out.println(response.getEntity(String.class));
- System.out.println("End of LogType:" + logFile);
- } catch (ClientHandlerException | UniformInterfaceException ex) {
- System.err.println("Can not find the log file:" + logFile
- + " for the container:" + containerIdStr + " in NodeManager:"
- + nodeId);
+ String nodeHttpAddress = request.getNodeHttpAddress();
+ String nodeId = request.getNodeId();
+ String appOwner = request.getAppOwner();
+ PrintStream out = logCliHelper.createPrintStream(localDir, nodeId,
+ containerIdStr);
+ try {
+ // fetch all the log files for the container
+ if (fetchAllLogFiles(logFiles)) {
+ requestedLogFiles =
+ getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
+ }
+ Client webServiceClient = Client.create();
+ String containerString = "\n\nContainer: " + containerIdStr;
+ out.println(containerString);
+ out.println(StringUtils.repeat("=", containerString.length()));
+
+ for (String logFile : requestedLogFiles) {
+ out.println("LogType:" + logFile);
+ out.println("Log Upload Time:"
+ + Times.format(System.currentTimeMillis()));
+ out.println("Log Contents:");
+ try {
+ WebResource webResource =
+ webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf)
+ + nodeHttpAddress);
+ ClientResponse response =
+ webResource.path("ws").path("v1").path("node")
+ .path("containerlogs").path(containerIdStr).path(logFile)
+ .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
+ out.println(response.getEntity(String.class));
+ out.println("End of LogType:" + logFile);
+ out.flush();
+ } catch (ClientHandlerException | UniformInterfaceException ex) {
+ System.err.println("Can not find the log file:" + logFile
+ + " for the container:" + containerIdStr + " in NodeManager:"
+ + nodeId);
+ }
}
+ } finally {
+ logCliHelper.closePrintStream(out);
}
// for the case, we have already uploaded partial logs in HDFS
logCliHelper.dumpAContainersLogsForALogType(appId, containerIdStr, nodeId,
- appOwner, Arrays.asList(requestedLogFiles), false);
+ appOwner, Arrays.asList(requestedLogFiles), false, localDir);
}
- private void printContainerLogsForFinishedApplication(String appId,
- String containerId, String nodeAddress, String[] logFiles,
- LogCLIHelpers logCliHelper, String appOwner) throws IOException {
- String containerString = "\n\nContainer: " + containerId;
- System.out.println(containerString);
- System.out.println(StringUtils.repeat("=", containerString.length()));
- logCliHelper.dumpAContainersLogsForALogType(appId, containerId,
- nodeAddress, appOwner, logFiles != null ? Arrays.asList(logFiles)
- : null);
+ private void printContainerLogsForFinishedApplication(
+ ContainerLogsRequest request, String[] logFiles,
+ LogCLIHelpers logCliHelper, String localDir)
+ throws IOException {
+ logCliHelper.dumpAContainersLogsForALogType(request.getAppId().toString(),
+ request.getContainerId().toString(), request.getNodeId(),
+ request.getAppOwner(), logFiles != null ? Arrays.asList(logFiles)
+ : null, localDir);
}
private int printContainerLogsForFinishedApplicationWithoutNodeId(
String appId, String containerId, String[] logFiles,
- LogCLIHelpers logCliHelper, String appOwner) throws IOException {
- String containerString = "\n\nContainer: " + containerId;
- System.out.println(containerString);
- System.out.println(StringUtils.repeat("=", containerString.length()));
+ LogCLIHelpers logCliHelper, String appOwner, String localDir)
+ throws IOException {
return logCliHelper.dumpAContainersLogsForALogTypeWithoutNodeId(appId,
containerId, appOwner, logFiles != null ?
- Arrays.asList(logFiles) : null);
+ Arrays.asList(logFiles) : null, localDir);
}
private ContainerReport getContainerReport(String containerIdStr)
@@ -433,36 +439,41 @@ public class LogsCLI extends Configured implements Tool {
|| appState == YarnApplicationState.KILLED;
}
- private int printAMContainerLogs(Configuration conf, String appId,
- List<String> amContainers, String[] logFiles, LogCLIHelpers logCliHelper,
- String appOwner, boolean applicationFinished) throws Exception {
+ private int printAMContainerLogs(Configuration conf,
+ ContainerLogsRequest request, List<String> amContainers,
+ String[] logFiles, LogCLIHelpers logCliHelper, String localDir)
+ throws Exception {
List<JSONObject> amContainersList = null;
- List<AMLogsRequest> requests = new ArrayList<AMLogsRequest>();
+ List<ContainerLogsRequest> requests =
+ new ArrayList<ContainerLogsRequest>();
boolean getAMContainerLists = false;
+ String appId = request.getAppId().toString();
String errorMessage = "";
try {
amContainersList = getAMContainerInfoForRMWebService(conf, appId);
if (amContainersList != null && !amContainersList.isEmpty()) {
getAMContainerLists = true;
for (JSONObject amContainer : amContainersList) {
- AMLogsRequest request = new AMLogsRequest(applicationFinished);
- request.setAmContainerId(amContainer.getString("containerId"));
- request.setNodeHttpAddress(amContainer.getString("nodeHttpAddress"));
- request.setNodeId(amContainer.getString("nodeId"));
- requests.add(request);
+ ContainerLogsRequest amRequest = new ContainerLogsRequest(request);
+ amRequest.setContainerId(amContainer.getString("containerId"));
+ amRequest.setNodeHttpAddress(
+ amContainer.getString("nodeHttpAddress"));
+ amRequest.setNodeId(amContainer.getString("nodeId"));
+ requests.add(amRequest);
}
}
} catch (Exception ex) {
errorMessage = ex.getMessage();
- if (applicationFinished) {
+ if (request.isAppFinished()) {
try {
amContainersList = getAMContainerInfoForAHSWebService(conf, appId);
if (amContainersList != null && !amContainersList.isEmpty()) {
getAMContainerLists = true;
for (JSONObject amContainer : amContainersList) {
- AMLogsRequest request = new AMLogsRequest(applicationFinished);
- request.setAmContainerId(amContainer.getString("amContainerId"));
- requests.add(request);
+ ContainerLogsRequest amRequest = new ContainerLogsRequest(
+ request);
+ amRequest.setContainerId(amContainer.getString("amContainerId"));
+ requests.add(amRequest);
}
}
} catch (Exception e) {
@@ -479,9 +490,9 @@ public class LogsCLI extends Configured implements Tool {
}
if (amContainers.contains("ALL")) {
- for (AMLogsRequest request : requests) {
- outputAMContainerLogs(request, conf, appId, logFiles, logCliHelper,
- appOwner);
+ for (ContainerLogsRequest amRequest : requests) {
+ outputAMContainerLogs(amRequest, conf, logFiles,
+ logCliHelper, localDir);
}
System.out.println();
System.out.println("Specified ALL for -am option. "
@@ -490,12 +501,12 @@ public class LogsCLI extends Configured implements Tool {
for (String amContainer : amContainers) {
int amContainerId = Integer.parseInt(amContainer.trim());
if (amContainerId == -1) {
- outputAMContainerLogs(requests.get(requests.size() - 1), conf, appId,
- logFiles, logCliHelper, appOwner);
+ outputAMContainerLogs(requests.get(requests.size() - 1), conf,
+ logFiles, logCliHelper, localDir);
} else {
if (amContainerId <= requests.size()) {
- outputAMContainerLogs(requests.get(amContainerId - 1), conf, appId,
- logFiles, logCliHelper, appOwner);
+ outputAMContainerLogs(requests.get(amContainerId - 1), conf,
+ logFiles, logCliHelper, localDir);
}
}
}
@@ -503,11 +514,11 @@ public class LogsCLI extends Configured implements Tool {
return 0;
}
- private void outputAMContainerLogs(AMLogsRequest request, Configuration conf,
- String appId, String[] logFiles, LogCLIHelpers logCliHelper,
- String appOwner) throws Exception {
+ private void outputAMContainerLogs(ContainerLogsRequest request,
+ Configuration conf, String[] logFiles,
+ LogCLIHelpers logCliHelper, String localDir) throws Exception {
String nodeHttpAddress = request.getNodeHttpAddress();
- String containerId = request.getAmContainerId();
+ String containerId = request.getContainerId();
String nodeId = request.getNodeId();
if (request.isAppFinished()) {
@@ -516,6 +527,7 @@ public class LogsCLI extends Configured implements Tool {
try {
nodeId =
getContainerReport(containerId).getAssignedNode().toString();
+ request.setNodeId(nodeId);
} catch (Exception ex) {
System.err.println(ex);
nodeId = null;
@@ -526,8 +538,8 @@ public class LogsCLI extends Configured implements Tool {
if(!fetchAllLogFiles(logFiles)) {
requestedLogFilesList = logFiles;
}
- printContainerLogsForFinishedApplication(appId, containerId, nodeId,
- requestedLogFilesList, logCliHelper, appOwner);
+ printContainerLogsForFinishedApplication(request,
+ requestedLogFilesList, logCliHelper, localDir);
}
}
} else {
@@ -540,36 +552,34 @@ public class LogsCLI extends Configured implements Tool {
getContainerLogFiles(getConf(), containerId, nodeHttpAddress);
}
printContainerLogsFromRunningApplication(conf,
- ContainerId.fromString(containerId), nodeHttpAddress, nodeId,
- requestedLogFiles, logCliHelper, appOwner);
+ request, requestedLogFiles, logCliHelper, localDir);
}
}
}
- private int showMetaInfo(YarnApplicationState appState,
- boolean appStateObtainedSuccessfully, LogCLIHelpers logCliHelper,
- ApplicationId appId, String containerIdStr, String nodeAddress,
- String appOwner) throws IOException {
- if (!isApplicationFinished(appState) && appStateObtainedSuccessfully) {
+ private int showMetaInfo(ContainerLogsRequest request,
+ LogCLIHelpers logCliHelper) throws IOException {
+ if (!request.isAppFinished()) {
System.err.println("The -show_meta_info command can be only used "
+ "with finished applications");
return -1;
} else {
- logCliHelper.printLogMetadata(appId, containerIdStr, nodeAddress,
- appOwner, System.out, System.err);
+ logCliHelper.printLogMetadata(request.getAppId(),
+ request.getContainerId(), request.getNodeId(),
+ request.getAppOwner(), System.out, System.err);
return 0;
}
}
- private int showNodeLists(YarnApplicationState appState,
- boolean appStateObtainedSuccessfully, LogCLIHelpers logCliHelper,
- ApplicationId appId, String appOwner) throws IOException {
- if (!isApplicationFinished(appState) && appStateObtainedSuccessfully) {
+ private int showNodeLists(ContainerLogsRequest request,
+ LogCLIHelpers logCliHelper) throws IOException {
+ if (!request.isAppFinished()) {
System.err.println("The -list_nodes command can be only used with "
+ "finished applications");
return -1;
} else {
- logCliHelper.printNodesList(appId, appOwner, System.out, System.err);
+ logCliHelper.printNodesList(request.getAppId(), request.getAppOwner(),
+ System.out, System.err);
return 0;
}
}
@@ -619,11 +629,15 @@ public class LogsCLI extends Configured implements Tool {
opts.addOption(LIST_NODES_OPTION, false,
"Show the list of nodes that successfully aggregated logs. "
+ "This option can only be used with finished applications.");
+ opts.addOption(OUT_OPTION, true, "Local directory for storing individual "
+ + "container logs. The container logs will be stored based on the "
+ + "node the container ran on.");
opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID");
opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID");
opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address");
opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner");
opts.getOption(AM_CONTAINER_OPTION).setArgName("AM Containers");
+ opts.getOption(OUT_OPTION).setArgName("Local Directory");
return opts;
}
@@ -637,6 +651,7 @@ public class LogsCLI extends Configured implements Tool {
printOpts.addOption(commandOpts.getOption(CONTAINER_LOG_FILES));
printOpts.addOption(commandOpts.getOption(SHOW_META_INFO));
printOpts.addOption(commandOpts.getOption(LIST_NODES_OPTION));
+ printOpts.addOption(commandOpts.getOption(OUT_OPTION));
return printOpts;
}
@@ -671,10 +686,9 @@ public class LogsCLI extends Configured implements Tool {
return amContainersList;
}
- private int fetchAMContainerLogs(String[] logFiles,
- YarnApplicationState appState, ApplicationId appId,
- String appOwner, List<String> amContainersList,
- LogCLIHelpers logCliHelper) throws Exception {
+ private int fetchAMContainerLogs(ContainerLogsRequest request,
+ List<String> amContainersList, String[] logFiles,
+ LogCLIHelpers logCliHelper, String localDir) throws Exception {
// if we do not specify the value for CONTAINER_LOG_FILES option,
// we will only output syslog
if (logFiles == null || logFiles.length == 0) {
@@ -685,10 +699,9 @@ public class LogsCLI extends Configured implements Tool {
// and containerId for all the AM Containers.
// After that, we will call NodeManager webService to get the
// related logs
- if (appState == YarnApplicationState.ACCEPTED
- || appState == YarnApplicationState.RUNNING) {
- return printAMContainerLogs(getConf(), appId.toString(), amContainersList,
- logFiles, logCliHelper, appOwner, false);
+ if (!request.isAppFinished()) {
+ return printAMContainerLogs(getConf(), request, amContainersList,
+ logFiles, logCliHelper, localDir);
} else {
// If the application is in the final state, we will call RM webservice
// to get all AppAttempts information first. If we get nothing,
@@ -698,9 +711,11 @@ public class LogsCLI extends Configured implements Tool {
// to get logs from HDFS directly.
if (getConf().getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
- return printAMContainerLogs(getConf(), appId.toString(),
- amContainersList, logFiles, logCliHelper, appOwner, true);
+ return printAMContainerLogs(getConf(), request, amContainersList,
+ logFiles, logCliHelper, localDir);
} else {
+ ApplicationId appId = request.getAppId();
+ String appOwner = request.getAppOwner();
System.err.println("Can not get AMContainers logs for "
+ "the application:" + appId + " with the appOwner:" + appOwner);
System.err.println("This application:" + appId + " has finished."
@@ -713,18 +728,18 @@ public class LogsCLI extends Configured implements Tool {
}
}
- private int fetchContainerLogs(YarnApplicationState appState,
- boolean appStateObtainedSuccessfully, String[] logFiles,
- String appOwner, String nodeAddress,
- ContainerId containerId, LogCLIHelpers logCliHelper) throws IOException {
+ private int fetchContainerLogs(ContainerLogsRequest request,
+ String[] logFiles, LogCLIHelpers logCliHelper, String localDir)
+ throws IOException {
int resultCode = 0;
- String appIdStr = containerId.getApplicationAttemptId()
- .getApplicationId().toString();
- String containerIdStr = containerId.toString();
+ String appIdStr = request.getAppId().toString();
+ String containerIdStr = request.getContainerId();
+ String nodeAddress = request.getNodeId();
+ String appOwner = request.getAppOwner();
+ boolean isAppFinished = request.isAppFinished();
// if we provide the node address and the application is in the final
// state, we could directly get logs from HDFS.
- if (nodeAddress != null && (!appStateObtainedSuccessfully ||
- isApplicationFinished(appState))) {
+ if (nodeAddress != null && isAppFinished) {
// if user specified "ALL" as the logFiles param, pass null
// to logCliHelper so that it fetches all the logs
List<String> logs;
@@ -736,7 +751,7 @@ public class LogsCLI extends Configured implements Tool {
logs = Arrays.asList(logFiles);
}
return logCliHelper.dumpAContainersLogsForALogType(appIdStr,
- containerIdStr, nodeAddress, appOwner, logs);
+ containerIdStr, nodeAddress, appOwner, logs, localDir);
}
String nodeHttpAddress = null;
String nodeId = null;
@@ -749,16 +764,18 @@ public class LogsCLI extends Configured implements Tool {
report.getNodeHttpAddress().replaceFirst(
WebAppUtils.getHttpSchemePrefix(getConf()), "");
nodeId = report.getAssignedNode().toString();
+ request.setNodeId(nodeId);
+ request.setNodeHttpAddress(nodeHttpAddress);
} catch (IOException | YarnException ex) {
- if (!appStateObtainedSuccessfully || isApplicationFinished(appState)) {
+ if (isAppFinished) {
String[] requestedLogFiles = logFiles;
if(fetchAllLogFiles(logFiles)) {
requestedLogFiles = null;
}
return printContainerLogsForFinishedApplicationWithoutNodeId(
appIdStr, containerIdStr, requestedLogFiles, logCliHelper,
- appOwner);
- } else if (!isApplicationFinished(appState)) {
+ appOwner, localDir);
+ } else {
System.err.println("Unable to get logs for this container:"
+ containerIdStr + "for the application:" + appIdStr
+ " with the appOwner: " + appOwner);
@@ -772,12 +789,12 @@ public class LogsCLI extends Configured implements Tool {
// If the application is not in the final state,
// we will provide the NodeHttpAddress and get the container logs
// by calling NodeManager webservice.
- if (!isApplicationFinished(appState)) {
+ if (!isAppFinished) {
if (logFiles == null || logFiles.length == 0) {
logFiles = new String[] {"syslog"};
}
- printContainerLogsFromRunningApplication(getConf(), containerId,
- nodeHttpAddress, nodeId, logFiles, logCliHelper, appOwner);
+ printContainerLogsFromRunningApplication(getConf(), request,
+ logFiles, logCliHelper, localDir);
} else {
String[] requestedLogFiles = logFiles;
if(fetchAllLogFiles(logFiles)) {
@@ -785,39 +802,95 @@ public class LogsCLI extends Configured implements Tool {
}
// If the application is in the final state, we will directly
// get the container logs from HDFS.
- printContainerLogsForFinishedApplication(appIdStr, containerIdStr,
- nodeId, requestedLogFiles, logCliHelper, appOwner);
+ printContainerLogsForFinishedApplication(request,
+ requestedLogFiles, logCliHelper, localDir);
}
return resultCode;
}
- private static class AMLogsRequest {
- private String amContainerId;
+ private int fetchApplicationLogs(ApplicationId appId, String appOwner,
+ LogCLIHelpers logCliHelper, String localDir) throws IOException {
+ int resultCode =
+ logCliHelper.dumpAllContainersLogs(appId, appOwner, localDir);
+ if (resultCode == -1) {
+ System.err.println("Can not find the logs for the application: "
+ + appId + " with the appOwner: " + appOwner);
+ }
+ return resultCode;
+ }
+
+ private String guessAppOwner(ApplicationReport appReport,
+ ApplicationId appId) throws IOException {
+ String appOwner = null;
+ if (appReport != null) {
+ //always use the app owner from the app report if possible
+ appOwner = appReport.getUser();
+ } else {
+ appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
+ appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(
+ appId, appOwner, getConf());
+ }
+ return appOwner;
+ }
+
+ private static class ContainerLogsRequest {
+ private ApplicationId appId;
+ private String containerId;
private String nodeId;
private String nodeHttpAddress;
- private final boolean isAppFinished;
+ private String appOwner;
+ private boolean appFinished;
+
+ public ContainerLogsRequest(ContainerLogsRequest request) {
+ this.setAppId(request.getAppId());
+ this.setAppFinished(request.isAppFinished());
+ this.setAppOwner(request.getAppOwner());
+ this.setNodeId(request.getNodeId());
+ this.setNodeHttpAddress(request.getNodeHttpAddress());
+ this.setContainerId(request.getContainerId());
+ }
+
+ public ContainerLogsRequest(ApplicationId applicationId,
+ boolean isAppFinished, String owner,
+ String address, String httpAddress, String container) {
+ this.setAppId(applicationId);
+ this.setAppFinished(isAppFinished);
+ this.setAppOwner(owner);
+ this.setNodeId(address);
+ this.setNodeHttpAddress(httpAddress);
+ this.setContainerId(container);
+ }
- AMLogsRequest(boolean isAppFinished) {
- this.isAppFinished = isAppFinished;
- this.setAmContainerId("");
- this.setNodeId("");
- this.setNodeHttpAddress("");
+ public ApplicationId getAppId() {
+ return appId;
}
- public String getAmContainerId() {
- return amContainerId;
+ public void setAppId(ApplicationId appId) {
+ this.appId = appId;
}
- public void setAmContainerId(String amContainerId) {
- this.amContainerId = amContainerId;
+ public String getContainerId() {
+ return containerId;
+ }
+
+ public void setContainerId(String containerId) {
+ this.containerId = containerId;
}
public String getNodeId() {
return nodeId;
}
- public void setNodeId(String nodeId) {
- this.nodeId = nodeId;
+ public void setNodeId(String nodeAddress) {
+ this.nodeId = nodeAddress;
+ }
+
+ public String getAppOwner() {
+ return appOwner;
+ }
+
+ public void setAppOwner(String appOwner) {
+ this.appOwner = appOwner;
}
public String getNodeHttpAddress() {
@@ -829,21 +902,11 @@ public class LogsCLI extends Configured implements Tool {
}
public boolean isAppFinished() {
- return isAppFinished;
+ return appFinished;
}
- }
- private String guessAppOwner(ApplicationReport appReport,
- ApplicationId appId) throws IOException {
- String appOwner = null;
- if (appReport != null) {
- //always use the app owner from the app report if possible
- appOwner = appReport.getUser();
- } else {
- appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
- appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(
- appId, appOwner, getConf());
+ public void setAppFinished(boolean appFinished) {
+ this.appFinished = appFinished;
}
- return appOwner;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef175779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 4f1c629..d649ce7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -26,11 +26,13 @@ import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.Writer;
@@ -41,8 +43,9 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
@@ -180,6 +183,10 @@ public class TestLogsCLI {
pw.println(" container log files. Use \"ALL\" to fetch");
pw.println(" all the log files for the container.");
pw.println(" -nodeAddress <Node Address> NodeAddress in the format nodename:port");
+ pw.println(" -out <Local Directory> Local directory for storing individual");
+ pw.println(" container logs. The container logs will");
+ pw.println(" be stored based on the node the container");
+ pw.println(" ran on.");
pw.println(" -show_meta_info Show the log metadata, including log-file");
pw.println(" names, the size of the log files. You can");
pw.println(" combine this with --containerId to get");
@@ -493,6 +500,103 @@ public class TestLogsCLI {
}
@Test (timeout = 15000)
+ public void testSaveContainerLogsLocally() throws Exception {
+ String remoteLogRootDir = "target/logs/";
+ String rootLogDir = "target/LocalLogs";
+ String localDir = "target/SaveLogs";
+ Path localPath = new Path(localDir);
+
+ Configuration configuration = new Configuration();
+ configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
+ configuration
+ .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
+ configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
+
+ FileSystem fs = FileSystem.get(configuration);
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ ContainerId containerId1 = ContainerId.newContainerId(
+ appAttemptId, 1);
+ ContainerId containerId2 = ContainerId.newContainerId(
+ appAttemptId, 2);
+ containerIds.add(containerId1);
+ containerIds.add(containerId2);
+
+ List<NodeId> nodeIds = new ArrayList<NodeId>();
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+ NodeId nodeId2 = NodeId.newInstance("test", 4567);
+ nodeIds.add(nodeId);
+ nodeIds.add(nodeId2);
+
+ try {
+ createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs,
+ appId, containerIds, nodeIds);
+
+ YarnClient mockYarnClient =
+ createMockYarnClient(YarnApplicationState.FINISHED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ LogsCLI cli = new LogsCLIForTest(mockYarnClient);
+ cli.setConf(configuration);
+ int exitCode = cli.run(new String[] {"-applicationId",
+ appId.toString(),
+ "-out" , localPath.toString()});
+ assertTrue(exitCode == 0);
+
+ // make sure we created a dir named as node id
+ FileStatus[] nodeDir = fs.listStatus(localPath);
+ Arrays.sort(nodeDir);
+ assertTrue(nodeDir.length == 2);
+ assertTrue(nodeDir[0].getPath().getName().contains(
+ LogAggregationUtils.getNodeString(nodeId)));
+ assertTrue(nodeDir[1].getPath().getName().contains(
+ LogAggregationUtils.getNodeString(nodeId2)));
+
+ FileStatus[] container1Dir = fs.listStatus(nodeDir[0].getPath());
+ assertTrue(container1Dir.length == 1);
+ assertTrue(container1Dir[0].getPath().getName().equals(
+ containerId1.toString()));
+ String container1= readContainerContent(container1Dir[0].getPath(), fs);
+ assertTrue(container1.contains("Hello " + containerId1
+ + " in syslog!"));
+
+ FileStatus[] container2Dir = fs.listStatus(nodeDir[1].getPath());
+ assertTrue(container2Dir.length == 1);
+ assertTrue(container2Dir[0].getPath().getName().equals(
+ containerId2.toString()));
+ String container2= readContainerContent(container2Dir[0].getPath(), fs);
+ assertTrue(container2.contains("Hello " + containerId2
+ + " in syslog!"));
+ } finally {
+ fs.delete(new Path(remoteLogRootDir), true);
+ fs.delete(new Path(rootLogDir), true);
+ fs.delete(localPath, true);
+ }
+ }
+
+ private String readContainerContent(Path containerPath,
+ FileSystem fs) throws IOException {
+ assertTrue(fs.exists(containerPath));
+ StringBuffer inputLine = new StringBuffer();
+ BufferedReader reader = null;
+ try {
+ reader = new BufferedReader(new InputStreamReader(
+ fs.open(containerPath)));
+ String tmp;
+ while ((tmp = reader.readLine()) != null) {
+ inputLine.append(tmp);
+ }
+ return inputLine.toString();
+ } finally {
+ if (reader != null) {
+ IOUtils.closeQuietly(reader);
+ }
+ }
+ }
+
+ @Test (timeout = 15000)
public void testPrintContainerLogMetadata() throws Exception {
String remoteLogRootDir = "target/logs/";
Configuration configuration = new Configuration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef175779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index ba0dd89..339df9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -24,8 +24,10 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.AccessDeniedException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.List;
-
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configurable;
@@ -54,7 +56,7 @@ public class LogCLIHelpers implements Configurable {
public int dumpAContainersLogs(String appId, String containerId,
String nodeId, String jobOwner) throws IOException {
return dumpAContainersLogsForALogType(appId, containerId, nodeId, jobOwner,
- null);
+ null, null);
}
@Private
@@ -107,17 +109,17 @@ public class LogCLIHelpers implements Configurable {
@Private
@VisibleForTesting
public int dumpAContainersLogsForALogType(String appId, String containerId,
- String nodeId, String jobOwner, List<String> logType)
+ String nodeId, String jobOwner, List<String> logType, String localDir)
throws IOException {
- return dumpAContainersLogsForALogType(appId, containerId, nodeId, jobOwner,
- logType, true);
+ return dumpAContainersLogsForALogType(appId, containerId, nodeId,
+ jobOwner, logType, true, localDir);
}
@Private
@VisibleForTesting
public int dumpAContainersLogsForALogType(String appId, String containerId,
String nodeId, String jobOwner, List<String> logType,
- boolean outputFailure) throws IOException {
+ boolean outputFailure, String localDir) throws IOException {
ApplicationId applicationId = ConverterUtils.toApplicationId(appId);
RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
applicationId, jobOwner);
@@ -137,17 +139,21 @@ public class LogCLIHelpers implements Configurable {
if (fileName.contains(LogAggregationUtils.getNodeString(nodeId))
&& !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
AggregatedLogFormat.LogReader reader = null;
+ PrintStream out = createPrintStream(localDir, fileName, containerId);
try {
+ String containerString = "\n\nContainer: " + containerId;
+ out.println(containerString);
+ out.println(StringUtils.repeat("=", containerString.length()));
reader =
new AggregatedLogFormat.LogReader(getConf(),
thisNodeFile.getPath());
if (logType == null) {
- if (dumpAContainerLogs(containerId, reader, System.out,
+ if (dumpAContainerLogs(containerId, reader, out,
thisNodeFile.getModificationTime()) > -1) {
foundContainerLogs = true;
}
} else {
- if (dumpAContainerLogsForALogType(containerId, reader, System.out,
+ if (dumpAContainerLogsForALogType(containerId, reader, out,
thisNodeFile.getModificationTime(), logType) > -1) {
foundContainerLogs = true;
}
@@ -156,11 +162,14 @@ public class LogCLIHelpers implements Configurable {
if (reader != null) {
reader.close();
}
+ closePrintStream(out);
}
}
}
- if (!foundContainerLogs && outputFailure) {
- containerLogNotFound(containerId);
+ if (!foundContainerLogs) {
+ if (outputFailure) {
+ containerLogNotFound(containerId);
+ }
return -1;
}
return 0;
@@ -168,8 +177,8 @@ public class LogCLIHelpers implements Configurable {
@Private
public int dumpAContainersLogsForALogTypeWithoutNodeId(String appId,
- String containerId, String jobOwner, List<String> logType)
- throws IOException {
+ String containerId, String jobOwner, List<String> logType,
+ String localDir) throws IOException {
ApplicationId applicationId = ConverterUtils.toApplicationId(appId);
RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
applicationId, jobOwner);
@@ -182,17 +191,28 @@ public class LogCLIHelpers implements Configurable {
if (!thisNodeFile.getPath().getName().endsWith(
LogAggregationUtils.TMP_FILE_SUFFIX)) {
AggregatedLogFormat.LogReader reader = null;
+ PrintStream out = System.out;
try {
reader =
new AggregatedLogFormat.LogReader(getConf(),
thisNodeFile.getPath());
+ if (getContainerLogsStream(containerId, reader) == null) {
+ continue;
+ }
+ reader =
+ new AggregatedLogFormat.LogReader(getConf(),
+ thisNodeFile.getPath());
+ out = createPrintStream(localDir, thisNodeFile.getPath().getName(),
+ containerId);
+ out.println(containerId);
+ out.println(StringUtils.repeat("=", containerId.length()));
if (logType == null) {
- if (dumpAContainerLogs(containerId, reader, System.out,
+ if (dumpAContainerLogs(containerId, reader, out,
thisNodeFile.getModificationTime()) > -1) {
foundContainerLogs = true;
}
} else {
- if (dumpAContainerLogsForALogType(containerId, reader, System.out,
+ if (dumpAContainerLogsForALogType(containerId, reader, out,
thisNodeFile.getModificationTime(), logType) > -1) {
foundContainerLogs = true;
}
@@ -201,6 +221,7 @@ public class LogCLIHelpers implements Configurable {
if (reader != null) {
reader.close();
}
+ closePrintStream(out);
}
}
}
@@ -210,19 +231,13 @@ public class LogCLIHelpers implements Configurable {
}
return 0;
}
+
@Private
public int dumpAContainerLogs(String containerIdStr,
AggregatedLogFormat.LogReader reader, PrintStream out,
long logUploadedTime) throws IOException {
- DataInputStream valueStream;
- LogKey key = new LogKey();
- valueStream = reader.next(key);
-
- while (valueStream != null && !key.toString().equals(containerIdStr)) {
- // Next container
- key = new LogKey();
- valueStream = reader.next(key);
- }
+ DataInputStream valueStream = getContainerLogsStream(
+ containerIdStr, reader);
if (valueStream == null) {
return -1;
@@ -244,10 +259,8 @@ public class LogCLIHelpers implements Configurable {
return -1;
}
- @Private
- public int dumpAContainerLogsForALogType(String containerIdStr,
- AggregatedLogFormat.LogReader reader, PrintStream out,
- long logUploadedTime, List<String> logType) throws IOException {
+ private DataInputStream getContainerLogsStream(String containerIdStr,
+ AggregatedLogFormat.LogReader reader) throws IOException {
DataInputStream valueStream;
LogKey key = new LogKey();
valueStream = reader.next(key);
@@ -257,7 +270,15 @@ public class LogCLIHelpers implements Configurable {
key = new LogKey();
valueStream = reader.next(key);
}
+ return valueStream;
+ }
+ @Private
+ public int dumpAContainerLogsForALogType(String containerIdStr,
+ AggregatedLogFormat.LogReader reader, PrintStream out,
+ long logUploadedTime, List<String> logType) throws IOException {
+ DataInputStream valueStream = getContainerLogsStream(
+ containerIdStr, reader);
if (valueStream == null) {
return -1;
}
@@ -283,7 +304,7 @@ public class LogCLIHelpers implements Configurable {
@Private
public int dumpAllContainersLogs(ApplicationId appId, String appOwner,
- PrintStream out) throws IOException {
+ String localDir) throws IOException {
RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
appId, appOwner);
if (nodeFiles == null) {
@@ -310,20 +331,25 @@ public class LogCLIHelpers implements Configurable {
valueStream = reader.next(key);
while (valueStream != null) {
-
- String containerString =
- "\n\nContainer: " + key + " on "
- + thisNodeFile.getPath().getName();
- out.println(containerString);
- out.println(StringUtils.repeat("=", containerString.length()));
- while (true) {
- try {
- LogReader.readAContainerLogsForALogType(valueStream, out,
- thisNodeFile.getModificationTime());
- foundAnyLogs = true;
- } catch (EOFException eof) {
- break;
+ PrintStream out = createPrintStream(localDir,
+ thisNodeFile.getPath().getName(), key.toString());
+ try {
+ String containerString =
+ "\n\nContainer: " + key + " on "
+ + thisNodeFile.getPath().getName();
+ out.println(containerString);
+ out.println(StringUtils.repeat("=", containerString.length()));
+ while (true) {
+ try {
+ LogReader.readAContainerLogsForALogType(valueStream, out,
+ thisNodeFile.getModificationTime());
+ foundAnyLogs = true;
+ } catch (EOFException eof) {
+ break;
+ }
}
+ } finally {
+ closePrintStream(out);
}
// Next container
@@ -501,4 +527,24 @@ public class LogCLIHelpers implements Configurable {
+ "have permission to access " + remoteAppLogDir
+ ". Error message found: " + errorMessage);
}
+
+ @Private
+ public PrintStream createPrintStream(String localDir, String nodeId,
+ String containerId) throws IOException {
+ PrintStream out = System.out;
+ if(localDir != null && !localDir.isEmpty()) {
+ Path nodePath = new Path(localDir, LogAggregationUtils
+ .getNodeString(nodeId));
+ Files.createDirectories(Paths.get(nodePath.toString()));
+ Path containerLogPath = new Path(nodePath, containerId);
+ out = new PrintStream(containerLogPath.toString(), "UTF-8");
+ }
+ return out;
+ }
+
+ public void closePrintStream(PrintStream out) {
+ if (out != System.out) {
+ IOUtils.closeQuietly(out);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HDFS-10417. Improve error message
from checkBlockLocalPathAccess. Contributed by Tianyin Xu.
Posted by ar...@apache.org.
HDFS-10417. Improve error message from checkBlockLocalPathAccess. Contributed by Tianyin Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0942954e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0942954e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0942954e
Branch: refs/heads/HDFS-7240
Commit: 0942954e8ab6bef636c2648d0d5a0227a50f799c
Parents: fa3bc34
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue May 17 15:06:25 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue May 17 15:06:25 2016 -0500
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/datanode/DataNode.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0942954e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 288fc3e..410472e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1717,7 +1717,8 @@ public class DataNode extends ReconfigurableBase
throw new AccessControlException(
"Can't continue with getBlockLocalPathInfo() "
+ "authorization. The user " + currentUser
- + " is not allowed to call getBlockLocalPathInfo");
+ + " is not configured in "
+ + DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: MAPREDUCE-6686. Add a way to
download the job config from the mapred CLI (rkanter)
Posted by ar...@apache.org.
MAPREDUCE-6686. Add a way to download the job config from the mapred CLI (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/992a4935
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/992a4935
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/992a4935
Branch: refs/heads/HDFS-7240
Commit: 992a49353f0bdbc71534bfeeb134112c5624b94b
Parents: f04c81c
Author: Robert Kanter <rk...@apache.org>
Authored: Wed May 18 12:16:21 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed May 18 12:16:21 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/mapreduce/tools/CLI.java | 35 ++++++++++++++++++--
.../src/site/markdown/MapredCommands.md | 3 +-
.../hadoop/mapreduce/TestMRJobClient.java | 28 ++++++++++++++++
3 files changed, 63 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/992a4935/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
index 4f55792..924af58 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TIPStatus;
@@ -106,6 +108,7 @@ public class CLI extends Configured implements Tool {
int fromEvent = 0;
int nEvents = 0;
int jpvalue = 0;
+ String configOutFile = null;
boolean getStatus = false;
boolean getCounter = false;
boolean killJob = false;
@@ -121,6 +124,7 @@ public class CLI extends Configured implements Tool {
boolean failTask = false;
boolean setJobPriority = false;
boolean logs = false;
+ boolean downloadConfig = false;
if ("-submit".equals(cmd)) {
if (argv.length != 2) {
@@ -295,6 +299,14 @@ public class CLI extends Configured implements Tool {
displayUsage(cmd);
return exitCode;
}
+ } else if ("-config".equals(cmd)) {
+ downloadConfig = true;
+ if (argv.length != 3) {
+ displayUsage(cmd);
+ return exitCode;
+ }
+ jobid = argv[1];
+ configOutFile = argv[2];
} else {
displayUsage(cmd);
return exitCode;
@@ -473,6 +485,22 @@ public class CLI extends Configured implements Tool {
System.out.println(e.getMessage());
}
}
+ } else if (downloadConfig) {
+ Job job = getJob(JobID.forName(jobid));
+ if (job == null) {
+ System.out.println("Could not find job " + jobid);
+ } else {
+ String jobFile = job.getJobFile();
+ if (jobFile == null || jobFile.isEmpty()) {
+ System.out.println("Config file for job " + jobFile +
+ " could not be found.");
+ } else {
+ Path configPath = new Path(jobFile);
+ FileSystem fs = FileSystem.get(getConf());
+ fs.copyToLocalFile(configPath, new Path(configOutFile));
+ exitCode = 0;
+ }
+ }
}
} catch (RemoteException re) {
IOException unwrappedException = re.unwrapRemoteException();
@@ -549,7 +577,9 @@ public class CLI extends Configured implements Tool {
} else if ("-logs".equals(cmd)) {
System.err.println(prefix + "[" + cmd +
" <job-id> <task-attempt-id>]. " +
- " <task-attempt-id> is optional to get task attempt logs.");
+ " <task-attempt-id> is optional to get task attempt logs.");
+ } else if ("-config".equals(cmd)) {
+ System.err.println(prefix + "[" + cmd + " <job-id> <file>]");
} else {
System.err.printf(prefix + "<command> <args>%n");
System.err.printf("\t[-submit <job-file>]%n");
@@ -571,7 +601,8 @@ public class CLI extends Configured implements Tool {
"Valid values for <task-state> are " + taskStates);
System.err.printf("\t[-kill-task <task-attempt-id>]%n");
System.err.printf("\t[-fail-task <task-attempt-id>]%n");
- System.err.printf("\t[-logs <job-id> <task-attempt-id>]%n%n");
+ System.err.printf("\t[-logs <job-id> <task-attempt-id>]%n");
+ System.err.printf("\t[-config <job-id> <file>%n%n");
ToolRunner.printGenericCommandUsage(System.out);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/992a4935/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
index 64589d9..f312d31 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
@@ -81,7 +81,7 @@ Copy file or directories recursively. More information can be found at
Command to interact with Map Reduce Jobs.
-Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobHistoryFile|jobId> [-outfile <file>] [-format <human|json>]] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>]`
+Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobHistoryFile|jobId> [-outfile <file>] [-format <human|json>]] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>] [-config <job-id> <file>]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
@@ -99,6 +99,7 @@ Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id
| -list-blacklisted-trackers | List the black listed task trackers in the cluster. This command is not supported in MRv2 based cluster. |
| -list-attempt-ids *job-id* *task-type* *task-state* | List the attempt-ids based on the task type and the status given. Valid values for task-type are REDUCE, MAP. Valid values for task-state are running, pending, completed, failed, killed. |
| -logs *job-id* *task-attempt-id* | Dump the container log for a job if taskAttemptId is not specified, otherwise dump the log for the task with the specified taskAttemptId. The logs will be dumped in system out. |
+| -config *job-id* *file* | Download the job configuration file. |
### `pipes`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/992a4935/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
index 8fe9078..3a33bb1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
@@ -168,6 +168,8 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
testfailTask(conf);
// kill job
testKillJob(conf);
+ // download job config
+ testConfig(jobId, conf);
}
/**
@@ -535,6 +537,32 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
}
/**
+ * download job config
+ */
+ private void testConfig(String jobId, Configuration conf) throws Exception {
+ CLI jc = createJobClient();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+ // bad arguments
+ int exitCode = runTool(conf, jc, new String[] { "-config" }, out);
+ assertEquals("Exit code", -1, exitCode);
+ exitCode = runTool(conf, jc, new String[] { "-config job_invalid foo.xml" },
+ out);
+ assertEquals("Exit code", -1, exitCode);
+
+ // good arguments
+ File outFile = File.createTempFile("config", ".xml");
+ exitCode = runTool(conf, jc, new String[] { "-config", jobId,
+ outFile.toString()}, out);
+ assertEquals("Exit code", 0, exitCode);
+ BufferedReader br = new BufferedReader(new FileReader(outFile));
+ String line = br.readLine();
+ br.close();
+ assertEquals("<?xml version=\"1.0\" encoding=\"UTF-8\" " +
+ "standalone=\"no\"?><configuration>", line);
+ }
+
+ /**
* print job events list
*/
private void testJobEvents(String jobId, Configuration conf) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HADOOP-12930. Dynamic subcommands
for hadoop shell scripts (aw)
Posted by ar...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-gridmix/src/main/shellprofile.d/hadoop-gridmix.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/shellprofile.d/hadoop-gridmix.sh b/hadoop-tools/hadoop-gridmix/src/main/shellprofile.d/hadoop-gridmix.sh
new file mode 100755
index 0000000..b7887ba
--- /dev/null
+++ b/hadoop-tools/hadoop-gridmix/src/main/shellprofile.d/hadoop-gridmix.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f hadoop_subcommand_gridmix >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+ hadoop_add_subcommand "gridmix" "submit a mix of synthetic job, modeling a profiled from production load"
+ fi
+
+## @description gridmix command for hadoop
+## @audience public
+## @stability stable
+## @replaceable yes
+function hadoop_subcommand_gridmix
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.mapred.gridmix.Gridmix
+ hadoop_add_to_classpath_tools hadoop-rumen
+ hadoop_add_to_classpath_tools hadoop-gridmix
+}
+
+fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh b/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
new file mode 100755
index 0000000..d7d4022
--- /dev/null
+++ b/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f hadoop_subcommand_rumenfolder >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+ hadoop_add_subcommand "rumenfolder" "scale a rumen input trace"
+ fi
+
+## @description rumenfolder command for hadoop
+## @audience public
+## @stability stable
+## @replaceable yes
+function hadoop_subcommand_rumenfolder
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.rumen.Folder
+ hadoop_add_to_classpath_tools hadoop-rumen
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+fi
+
+if ! declare -f hadoop_subcommand_rumentrace >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+ hadoop_add_subcommand "rumentrace" "convert logs into a rumen trace"
+ fi
+
+## @description rumentrace command for hadoop
+## @audience public
+## @stability stable
+## @replaceable yes
+function hadoop_subcommand_rumentrace
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.rumen.TraceBuilder
+ hadoop_add_to_classpath_tools hadoop-rumen
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm b/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
index bee976a..34dfd0b 100644
--- a/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
+++ b/hadoop-tools/hadoop-rumen/src/site/markdown/Rumen.md.vm
@@ -50,8 +50,8 @@ but a simulation of the scheduler elects to run that task on a remote
rack, the simulator requires a runtime its input cannot provide.
To fill in these gaps, Rumen performs a statistical analysis of the
digest to estimate the variables the trace doesn't supply. Rumen traces
-drive both Gridmix (a benchmark of Hadoop MapReduce clusters) and Mumak
-(a simulator for the JobTracker).
+drive both Gridmix (a benchmark of Hadoop MapReduce clusters) and SLS
+(a simulator for the resource manager scheduler).
$H3 Motivation
@@ -126,16 +126,13 @@ can use the `Folder` utility to fold the current trace to the
desired length. The remaining part of this section explains these
utilities in detail.
-Examples in this section assumes that certain libraries are present
-in the java CLASSPATH. See [Dependencies](#Dependencies) for more details.
-
$H3 Trace Builder
$H4 Command
```
-java org.apache.hadoop.tools.rumen.TraceBuilder [options] <jobtrace-output> <topology-output> <inputs>
+hadoop rumentrace [options] <jobtrace-output> <topology-output> <inputs>
```
This command invokes the `TraceBuilder` utility of *Rumen*.
@@ -205,12 +202,8 @@ $H4 Options
$H4 Example
-*Rumen* expects certain library *JARs* to be present in the *CLASSPATH*.
-One simple way to run Rumen is to use
-`$HADOOP_HOME/bin/hadoop jar` command to run it as example below.
-
```
-java org.apache.hadoop.tools.rumen.TraceBuilder \
+hadoop rumentrace \
file:///tmp/job-trace.json \
file:///tmp/job-topology.json \
hdfs:///tmp/hadoop-yarn/staging/history/done_intermediate/testuser
@@ -229,7 +222,7 @@ $H3 Folder
$H4 Command
```
-java org.apache.hadoop.tools.rumen.Folder [options] [input] [output]
+hadoop rumenfolder [options] [input] [output]
```
This command invokes the `Folder` utility of
@@ -350,7 +343,7 @@ $H4 Examples
$H5 Folding an input trace with 10 hours of total runtime to generate an output trace with 1 hour of total runtime
```
-java org.apache.hadoop.tools.rumen.Folder \
+hadoop rumenfolder \
-output-duration 1h \
-input-cycle 20m \
file:///tmp/job-trace.json \
@@ -362,7 +355,7 @@ If the folded jobs are out of order then the command will bail out.
$H5 Folding an input trace with 10 hours of total runtime to generate an output trace with 1 hour of total runtime and tolerate some skewness
```
-java org.apache.hadoop.tools.rumen.Folder \
+hadoop rumenfolder \
-output-duration 1h \
-input-cycle 20m \
-allow-missorting \
@@ -378,7 +371,7 @@ If the folded jobs are out of order, then atmost
$H5 Folding an input trace with 10 hours of total runtime to generate an output trace with 1 hour of total runtime in debug mode
```
-java org.apache.hadoop.tools.rumen.Folder \
+hadoop rumenfolder \
-output-duration 1h \
-input-cycle 20m \
-debug -temp-directory file:///tmp/debug \
@@ -395,7 +388,7 @@ up.
$H5 Folding an input trace with 10 hours of total runtime to generate an output trace with 1 hour of total runtime with custom concentration.
```
-java org.apache.hadoop.tools.rumen.Folder \
+hadoop rumenfolder \
-output-duration 1h \
-input-cycle 20m \
-concentration 2 \
@@ -421,18 +414,3 @@ Look at the MapReduce
<a href="https://issues.apache.org/jira/browse/MAPREDUCE/component/12313617">rumen-component</a>
for further details.
-
-$H3 Dependencies
-
-*Rumen* expects certain library *JARs* to be present in the *CLASSPATH*.
-One simple way to run Rumen is to use
-`hadoop jar` command to run it as example below.
-
-```
-$HADOOP_HOME/bin/hadoop jar \
- $HADOOP_HOME/share/hadoop/tools/lib/hadoop-rumen-2.5.1.jar \
- org.apache.hadoop.tools.rumen.TraceBuilder \
- file:///tmp/job-trace.json \
- file:///tmp/job-topology.json \
- hdfs:///tmp/hadoop-yarn/staging/history/done_intermediate/testuser
-```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
index 5a07cc3..ffddc7c 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/DumpTypedBytes.java
@@ -91,8 +91,7 @@ public class DumpTypedBytes implements Tool {
}
private void printUsage() {
- System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
- + " dumptb <glob-pattern>");
+ System.out.println("Usage: mapred streaming dumptb <glob-pattern>");
System.out.println(" Dumps all files that match the given pattern to " +
"standard output as typed bytes.");
System.out.println(" The files can be text or sequence files");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
index eabf46c..92f9d03 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/HadoopStreaming.java
@@ -56,8 +56,7 @@ public class HadoopStreaming {
}
private static void printUsage() {
- System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
- + " [options]");
+ System.out.println("Usage: mapred streaming [options]");
System.out.println("Options:");
System.out.println(" dumptb <glob-pattern> Dumps all files that match the"
+ " given pattern to ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
index a7a001c..838cfa1 100644
--- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
+++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/LoadTypedBytes.java
@@ -89,8 +89,7 @@ public class LoadTypedBytes implements Tool {
}
private void printUsage() {
- System.out.println("Usage: $HADOOP_HOME/bin/hadoop jar hadoop-streaming.jar"
- + " loadtb <path>");
+ System.out.println("Usage: mapred streaming loadtb <path>");
System.out.println(" Reads typed bytes from standard input" +
" and stores them in a sequence file in");
System.out.println(" the specified path");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh b/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
new file mode 100755
index 0000000..cca016d
--- /dev/null
+++ b/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f mapred_subcommand_streaming >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+ hadoop_add_subcommand "streaming" "launch a mapreduce streaming job"
+ fi
+
+## @description streaming command for mapred
+## @audience public
+## @stability stable
+## @replaceable yes
+function mapred_subcommand_streaming
+{
+ declare jarname
+ declare oldifs
+
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
+ hadoop_add_to_classpath_tools hadoop-streaming
+
+ # locate the streaming jar so we have something to
+ # give to RunJar
+ oldifs=${IFS}
+ IFS=:
+ for jarname in ${CLASSPATH}; do
+ if [[ "${jarname}" =~ hadoop-streaming-[0-9] ]]; then
+ HADOOP_SUBCMD_ARGS=("${jarname}" "${HADOOP_SUBCMD_ARGS[@]}")
+ break
+ fi
+ done
+
+ IFS=${oldifs}
+
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+
+}
+
+fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm b/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
index cc8ed69..072a68b 100644
--- a/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
+++ b/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
@@ -62,7 +62,7 @@ Hadoop Streaming
Hadoop streaming is a utility that comes with the Hadoop distribution. The utility allows you to create and run Map/Reduce jobs with any executable or script as the mapper and/or the reducer. For example:
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-input myInputDirs \
-output myOutputDir \
-mapper /bin/cat \
@@ -88,7 +88,7 @@ Streaming supports streaming command options as well as [generic command options
**Note:** Be sure to place the generic options before the streaming options, otherwise the command will fail. For an example, see [Making Archives Available to Tasks](#Making_Archives_Available_to_Tasks).
- hadoop command [genericOptions] [streamingOptions]
+ mapred streaming [genericOptions] [streamingOptions]
The Hadoop streaming command options are listed here:
@@ -115,7 +115,7 @@ $H3 Specifying a Java Class as the Mapper/Reducer
You can supply a Java class as the mapper and/or the reducer.
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-input myInputDirs \
-output myOutputDir \
-inputformat org.apache.hadoop.mapred.KeyValueTextInputFormat \
@@ -128,7 +128,7 @@ $H3 Packaging Files With Job Submissions
You can specify any executable as the mapper and/or the reducer. The executables do not need to pre-exist on the machines in the cluster; however, if they don't, you will need to use "-file" option to tell the framework to pack your executable files as a part of job submission. For example:
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-input myInputDirs \
-output myOutputDir \
-mapper myPythonScript.py \
@@ -139,7 +139,7 @@ The above example specifies a user defined Python executable as the mapper. The
In addition to executable files, you can also package other auxiliary files (such as dictionaries, configuration files, etc) that may be used by the mapper and/or the reducer. For example:
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-input myInputDirs \
-output myOutputDir \
-mapper myPythonScript.py \
@@ -216,7 +216,7 @@ $H4 Specifying the Number of Reducers
To specify the number of reducers, for example two, use:
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-D mapreduce.job.reduces=2 \
-input myInputDirs \
-output myOutputDir \
@@ -229,7 +229,7 @@ As noted earlier, when the Map/Reduce framework reads a line from the stdout of
However, you can customize this default. You can specify a field separator other than the tab character (the default), and you can specify the nth (n \>= 1) character rather than the first character in a line (the default) as the separator between the key and value. For example:
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-D stream.map.output.field.separator=. \
-D stream.num.map.output.key.fields=4 \
-input myInputDirs \
@@ -279,7 +279,7 @@ User can specify a different symlink name for -archives using \#.
In this example, the input.txt file has two lines specifying the names of the two files: cachedir.jar/cache.txt and cachedir.jar/cache2.txt. "cachedir.jar" is a symlink to the archived directory, which has the files "cache.txt" and "cache2.txt".
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-archives 'hdfs://hadoop-nn1.example.com/user/me/samples/cachefile/cachedir.jar' \
-D mapreduce.job.maps=1 \
-D mapreduce.job.reduces=1 \
@@ -325,7 +325,7 @@ $H3 Hadoop Partitioner Class
Hadoop has a library class, [KeyFieldBasedPartitioner](../api/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.html), that is useful for many applications. This class allows the Map/Reduce framework to partition the map outputs based on certain key fields, not the whole keys. For example:
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-D stream.map.output.field.separator=. \
-D stream.num.map.output.key.fields=4 \
-D map.output.key.field.separator=. \
@@ -375,7 +375,7 @@ $H3 Hadoop Comparator Class
Hadoop has a library class, [KeyFieldBasedComparator](../api/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.html), that is useful for many applications. This class provides a subset of features provided by the Unix/GNU Sort. For example:
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-D mapreduce.job.output.key.comparator.class=org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator \
-D stream.map.output.field.separator=. \
-D stream.num.map.output.key.fields=4 \
@@ -411,7 +411,7 @@ Hadoop has a library package called [Aggregate](../api/org/apache/hadoop/mapred/
To use Aggregate, simply specify "-reducer aggregate":
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-input myInputDirs \
-output myOutputDir \
-mapper myAggregatorForKeyCount.py \
@@ -444,7 +444,7 @@ $H3 Hadoop Field Selection Class
Hadoop has a library class, [FieldSelectionMapReduce](../api/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.html), that effectively allows you to process text data like the unix "cut" utility. The map function defined in the class treats each input key/value pair as a list of fields. You can specify the field separator (the default is the tab character). You can select an arbitrary list of fields as the map output key, and an arbitrary list of fields as the map output value. Similarly, the reduce function defined in the class treats each input key/value pair as a list of fields. You can select an arbitrary list of fields as the reduce output key, and an arbitrary list of fields as the reduce output value. For example:
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-D mapreduce.map.output.key.field.separator=. \
-D mapreduce.partition.keypartitioner.options=-k1,2 \
-D mapreduce.fieldsel.data.field.separator=. \
@@ -495,7 +495,7 @@ Using an alias will not work, but variable substitution is allowed as shown in t
charlie 80
dan 75
- $ c2='cut -f2'; hadoop jar hadoop-streaming-${project.version}.jar \
+ $ c2='cut -f2'; mapred streaming \
-D mapreduce.job.name='Experiment' \
-input /user/me/samples/student_marks \
-output /user/me/samples/student_out \
@@ -525,7 +525,7 @@ $H3 How do I specify multiple input directories?
You can specify multiple input directories with multiple '-input' options:
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-input '/user/foo/dir1' -input '/user/foo/dir2' \
(rest of the command)
@@ -541,7 +541,7 @@ $H3 How do I parse XML documents using streaming?
You can use the record reader StreamXmlRecordReader to process XML documents.
- hadoop jar hadoop-streaming-${project.version}.jar \
+ mapred streaming \
-inputreader "StreamXmlRecord,begin=BEGIN_STRING,end=END_STRING" \
(rest of the command)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index cac3bb6..878c95f 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -16,7 +16,12 @@
# limitations under the License.
MYNAME="${BASH_SOURCE-$0}"
+HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
+## @description build up the yarn command's usage text.
+## @audience public
+## @stability stable
+## @replaceable no
function hadoop_usage
{
hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
@@ -46,9 +51,180 @@ function hadoop_usage
hadoop_add_subcommand "timelineserver" "run the timeline server"
hadoop_add_subcommand "top" "view cluster information"
hadoop_add_subcommand "version" "print the version"
- hadoop_generate_usage "${MYNAME}" true
+ hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
}
+## @description Default command handler for yarn command
+## @audience public
+## @stability stable
+## @replaceable no
+## @param CLI arguments
+function yarncmd_case
+{
+ subcmd=$1
+ shift
+
+ case ${subcmd} in
+ application|applicationattempt|container)
+ HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ set -- "${subcmd}" "$@"
+ ;;
+ classpath)
+ hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
+ ;;
+ cluster)
+ HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ClusterCLI
+ hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ daemonlog)
+ HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ envvars)
+ echo "JAVA_HOME='${JAVA_HOME}'"
+ echo "HADOOP_YARN_HOME='${HADOOP_YARN_HOME}'"
+ echo "YARN_DIR='${YARN_DIR}'"
+ echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'"
+ echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
+ echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+ echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+ echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
+ exit 0
+ ;;
+ jar)
+ HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ historyserver)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
+ echo "Instead use the timelineserver command for it." 1>&2
+ echo "Starting the History Server anyway..." 1>&2
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+ ;;
+ logs)
+ HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.LogsCLI
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ node)
+ HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.NodeCLI
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ nodemanager)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
+ hadoop_debug "Append YARN_NODEMANAGER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_NODEMANAGER_OPTS}"
+ # Backwards compatibility
+ if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
+ HADOOP_HEAPSIZE_MAX="${YARN_NODEMANAGER_HEAPSIZE}"
+ fi
+ ;;
+ proxyserver)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
+ hadoop_debug "Append YARN_PROXYSERVER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_PROXYSERVER_OPTS}"
+ # Backwards compatibility
+ if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
+ # shellcheck disable=SC2034
+ HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}"
+ fi
+ ;;
+ queue)
+ HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ resourcemanager)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_RESOURCEMANAGER_OPTS}"
+ hadoop_debug "Append YARN_RESOURCEMANAGER_OPTS onto HADOOP_OPTS"
+ # Backwards compatibility
+ if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
+ # shellcheck disable=SC2034
+ HADOOP_HEAPSIZE_MAX="${YARN_RESOURCEMANAGER_HEAPSIZE}"
+ fi
+ ;;
+ rmadmin)
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ scmadmin)
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ sharedcachemanager)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
+ hadoop_debug "Append YARN_SHAREDCACHEMANAGER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_SHAREDCACHEMANAGER_OPTS}"
+ ;;
+ timelineserver)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
+ hadoop_debug "Append YARN_TIMELINESERVER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_TIMELINESERVER_OPTS}"
+ # Backwards compatibility
+ if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
+ # shellcheck disable=SC2034
+ HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
+ fi
+ ;;
+ version)
+ HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ top)
+ doNotSetCols=0
+ doNotSetRows=0
+ for i in "$@"; do
+ if [[ $i == "-cols" ]]; then
+ doNotSetCols=1
+ fi
+ if [[ $i == "-rows" ]]; then
+ doNotSetRows=1
+ fi
+ done
+ if [ $doNotSetCols == 0 ] && [ -n "${TERM}" ]; then
+ cols=$(tput cols)
+ if [ -n "$cols" ]; then
+ args=( $@ )
+ args=("${args[@]}" "-cols" "$cols")
+ set -- "${args[@]}"
+ fi
+ fi
+ if [ $doNotSetRows == 0 ] && [ -n "${TERM}" ]; then
+ rows=$(tput lines)
+ if [ -n "$rows" ]; then
+ args=( $@ )
+ args=("${args[@]}" "-rows" "$rows")
+ set -- "${args[@]}"
+ fi
+ fi
+ HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.TopCLI
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
+ *)
+ HADOOP_CLASSNAME="${subcmd}"
+ if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
+ hadoop_exit_with_usage 1
+ fi
+ ;;
+ esac
+}
# let's locate libexec...
if [[ -n "${HADOOP_HOME}" ]]; then
@@ -74,195 +250,71 @@ if [[ $# = 0 ]]; then
fi
# get arguments
-COMMAND=$1
+HADOOP_SUBCMD=$1
shift
-case "${COMMAND}" in
- application|applicationattempt|container)
- CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- set -- "${COMMAND}" "$@"
- ;;
- classpath)
- hadoop_do_classpath_subcommand CLASS "$@"
- ;;
- cluster)
- CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
- YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- daemonlog)
- CLASS=org.apache.hadoop.log.LogLevel
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- envvars)
- echo "JAVA_HOME='${JAVA_HOME}'"
- echo "HADOOP_YARN_HOME='${HADOOP_YARN_HOME}'"
- echo "YARN_DIR='${YARN_DIR}'"
- echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'"
- echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
- echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
- echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
- echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
- exit 0
- ;;
- jar)
- CLASS=org.apache.hadoop.util.RunJar
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- historyserver)
- supportdaemonization="true"
- echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
- echo "Instead use the timelineserver command for it." 1>&2
- echo "Starting the History Server anyway..." 1>&2
- CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
- ;;
- logs)
- CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- node)
- CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- nodemanager)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
- hadoop_debug "Append YARN_NODEMANAGER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_NODEMANAGER_OPTS}"
- # Backwards compatibility
- if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
- HADOOP_HEAPSIZE_MAX="${YARN_NODEMANAGER_HEAPSIZE}"
- fi
- ;;
- proxyserver)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
- hadoop_debug "Append YARN_PROXYSERVER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_PROXYSERVER_OPTS}"
- # Backwards compatibility
- if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
- HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}"
- fi
- ;;
- queue)
- CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- resourcemanager)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_RESOURCEMANAGER_OPTS}"
- hadoop_debug "Append YARN_RESOURCEMANAGER_OPTS onto HADOOP_OPTS"
- # Backwards compatibility
- if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
- HADOOP_HEAPSIZE_MAX="${YARN_RESOURCEMANAGER_HEAPSIZE}"
- fi
- ;;
- rmadmin)
- CLASS='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- scmadmin)
- CLASS='org.apache.hadoop.yarn.client.SCMAdmin'
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- sharedcachemanager)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
- hadoop_debug "Append YARN_SHAREDCACHEMANAGER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_SHAREDCACHEMANAGER_OPTS}"
- ;;
- timelineserver)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
- hadoop_debug "Append YARN_TIMELINESERVER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_TIMELINESERVER_OPTS}"
- # Backwards compatibility
- if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
- HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
- fi
- ;;
- version)
- CLASS=org.apache.hadoop.util.VersionInfo
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- top)
- doNotSetCols=0
- doNotSetRows=0
- for i in "$@"; do
- if [[ $i == "-cols" ]]; then
- doNotSetCols=1
- fi
- if [[ $i == "-rows" ]]; then
- doNotSetRows=1
- fi
- done
- if [ $doNotSetCols == 0 ] && [ -n "${TERM}" ]; then
- cols=$(tput cols)
- if [ -n "$cols" ]; then
- args=( $@ )
- args=("${args[@]}" "-cols" "$cols")
- set -- "${args[@]}"
- fi
- fi
- if [ $doNotSetRows == 0 ] && [ -n "${TERM}" ]; then
- rows=$(tput lines)
- if [ -n "$rows" ]; then
- args=( $@ )
- args=("${args[@]}" "-rows" "$rows")
- set -- "${args[@]}"
- fi
- fi
- CLASS=org.apache.hadoop.yarn.client.cli.TopCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
- ;;
- *)
- CLASS="${COMMAND}"
- if ! hadoop_validate_classname "${CLASS}"; then
- hadoop_exit_with_usage 1
- fi
- ;;
-esac
+HADOOP_SUBCMD_ARGS=("$@")
+
+if declare -f yarn_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
+ hadoop_debug "Calling dynamically: yarn_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}"
+ "yarn_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+else
+ yarncmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+fi
-hadoop_verify_user "${COMMAND}"
+hadoop_verify_user "${HADOOP_SUBCMD}"
if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
hadoop_common_slave_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
exit $?
fi
-daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
-daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
+if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
+ HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
+ hadoop_verify_secure_prereq
+ hadoop_setup_secure_service
+ priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err"
+ priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+ daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+else
+ daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+fi
if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
# shellcheck disable=SC2034
HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
- HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
+ # shellcheck disable=SC2034
+ HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
fi
hadoop_finalize
-if [[ -n "${supportdaemonization}" ]]; then
- if [[ -n "${secure_service}" ]]; then
- hadoop_secure_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" \
- "${CLASS}" "${daemon_pidfile}" "${daemon_outfile}" \
- "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@"
+if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then
+ if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
+ hadoop_secure_daemon_handler \
+ "${HADOOP_DAEMON_MODE}" \
+ "${HADOOP_SUBCMD}" \
+ "${HADOOP_CLASSNAME}" \
+ "${daemon_pidfile}" \
+ "${daemon_outfile}" \
+ "${priv_pidfile}" \
+ "${priv_outfile}" \
+ "${priv_errfile}" \
+ "${HADOOP_SUBCMD_ARGS[@]}"
else
- hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}" \
- "${daemon_pidfile}" "${daemon_outfile}" "$@"
+ hadoop_daemon_handler \
+ "${HADOOP_DAEMON_MODE}" \
+ "${HADOOP_SUBCMD}" \
+ "${HADOOP_CLASSNAME}" \
+ "${daemon_pidfile}" \
+ "${daemon_outfile}" \
+ "${HADOOP_SUBCMD_ARGS[@]}"
fi
exit $?
else
- hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
+ # shellcheck disable=SC2086
+ hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-yarn-project/hadoop-yarn/shellprofile.d/hadoop-yarn.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/shellprofile.d/hadoop-yarn.sh b/hadoop-yarn-project/hadoop-yarn/shellprofile.d/hadoop-yarn.sh
new file mode 100644
index 0000000..4602293
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/shellprofile.d/hadoop-yarn.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop_add_profile yarn
+
+function _yarn_hadoop_classpath
+{
+ local i
+ #
+ # get all of the yarn jars+config in the path
+ #
+ # developers
+ if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
+ for i in yarn-api yarn-common yarn-mapreduce yarn-master-worker \
+ yarn-server/yarn-server-nodemanager \
+ yarn-server/yarn-server-common \
+ yarn-server/yarn-server-resourcemanager; do
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/$i/target/classes"
+ done
+
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/build/test/classes"
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/build/tools"
+ fi
+
+ if [[ -d "${HADOOP_YARN_HOME}/${YARN_DIR}/webapps" ]]; then
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"
+ fi
+
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}"'/*'
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"'/*'
+}
+
+function _yarn_hadoop_finalize
+{
+ # Add YARN custom options to comamnd line in case someone actaully
+ # used these.
+ #
+ # Note that we are replacing ' ' with '\ ' so that when we exec
+ # stuff it works
+ #
+ local yld=$HADOOP_LOG_DIR
+ hadoop_translate_cygwin_path yld
+ hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${yld}"
+ hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE}"
+ local yhd=$HADOOP_YARN_HOME
+ hadoop_translate_cygwin_path yhd
+ hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${yhd}"
+ hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn.sh b/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn.sh
deleted file mode 100644
index 4aa20b1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-hadoop_add_profile yarn
-
-function _yarn_hadoop_classpath
-{
- local i
- #
- # get all of the yarn jars+config in the path
- #
- # developers
- if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
- for i in yarn-api yarn-common yarn-mapreduce yarn-master-worker \
- yarn-server/yarn-server-nodemanager \
- yarn-server/yarn-server-common \
- yarn-server/yarn-server-resourcemanager; do
- hadoop_add_classpath "${HADOOP_YARN_HOME}/$i/target/classes"
- done
-
- hadoop_add_classpath "${HADOOP_YARN_HOME}/build/test/classes"
- hadoop_add_classpath "${HADOOP_YARN_HOME}/build/tools"
- fi
-
- if [[ -d "${HADOOP_YARN_HOME}/${YARN_DIR}/webapps" ]]; then
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"
- fi
-
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}"'/*'
- hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"'/*'
-}
-
-function _yarn_hadoop_finalize
-{
- # Add YARN custom options to comamnd line in case someone actaully
- # used these.
- #
- # Note that we are replacing ' ' with '\ ' so that when we exec
- # stuff it works
- #
- local yld=$HADOOP_LOG_DIR
- hadoop_translate_cygwin_path yld
- hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${yld}"
- hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE}"
- local yhd=$HADOOP_YARN_HOME
- hadoop_translate_cygwin_path yhd
- hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${yhd}"
- hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: MAPREDUCE-6657. Job history server
can fail on startup when NameNode is in start phase. Contributed by Haibo
Chen.
Posted by ar...@apache.org.
MAPREDUCE-6657. Job history server can fail on startup when NameNode is in start phase. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6ef876f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6ef876f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6ef876f
Branch: refs/heads/HDFS-7240
Commit: f6ef876fe158a5334cad7075f1966573a1c4dec9
Parents: e24fe26
Author: Junping Du <ju...@apache.org>
Authored: Tue May 17 14:41:47 2016 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Tue May 17 14:41:47 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/NameNode.java | 4 ++
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 +-
.../hadoop-mapreduce-client-hs/pom.xml | 4 ++
.../mapreduce/v2/hs/HistoryFileManager.java | 20 ++++--
...HistoryFileManagerInitWithNonRunningDFS.java | 75 ++++++++++++++++++++
5 files changed, 100 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ef876f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 3fc4e37..ae7a937 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -516,6 +516,10 @@ public class NameNode extends ReconfigurableBase implements
return role.equals(that);
}
+ public static String composeNotStartedMessage(NamenodeRole role) {
+ return role + " still not started";
+ }
+
/**
* Given a configuration get the address of the lifeline RPC server.
* If the lifeline RPC is not configured returns null.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ef876f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 2501be9..a63edea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -2073,7 +2073,8 @@ class NameNodeRpcServer implements NamenodeProtocols {
private void checkNNStartup() throws IOException {
if (!this.nn.isStarted()) {
- throw new RetriableException(this.nn.getRole() + " still not started");
+ String message = NameNode.composeNotStartedMessage(this.nn.getRole());
+ throw new RetriableException(message);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ef876f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
index ce76165..6166e80 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
@@ -39,6 +39,10 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ef876f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 27e743c..a1f4014 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -55,6 +55,9 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapreduce.jobhistory.JobSummary;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -599,12 +602,19 @@ public class HistoryFileManager extends AbstractService {
}
/**
+ * Check if the NameNode is still not started yet as indicated by the
+ * exception type and message.
* DistributedFileSystem returns a RemoteException with a message stating
* SafeModeException in it. So this is only way to check it is because of
- * being in safe mode.
+ * being in safe mode. In addition, Name Node may have not started yet, in
+ * which case, the message contains "NameNode still not started".
*/
- private boolean isBecauseSafeMode(Throwable ex) {
- return ex.toString().contains("SafeModeException");
+ private boolean isNameNodeStillNotStarted(Exception ex) {
+ String nameNodeNotStartedMsg = NameNode.composeNotStartedMessage(
+ HdfsServerConstants.NamenodeRole.NAMENODE);
+ return ex.toString().contains("SafeModeException") ||
+ (ex instanceof RetriableException && ex.getMessage().contains(
+ nameNodeNotStartedMsg));
}
/**
@@ -631,7 +641,7 @@ public class HistoryFileManager extends AbstractService {
}
succeeded = false;
} catch (IOException e) {
- if (isBecauseSafeMode(e)) {
+ if (isNameNodeStillNotStarted(e)) {
succeeded = false;
if (logWait) {
LOG.info("Waiting for FileSystem at " +
@@ -661,7 +671,7 @@ public class HistoryFileManager extends AbstractService {
"to be available");
}
} catch (IOException e) {
- if (isBecauseSafeMode(e)) {
+ if (isNameNodeStillNotStarted(e)) {
succeeded = false;
if (logWait) {
LOG.info("Waiting for FileSystem at " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ef876f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManagerInitWithNonRunningDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManagerInitWithNonRunningDFS.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManagerInitWithNonRunningDFS.java
new file mode 100644
index 0000000..d0fefef
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManagerInitWithNonRunningDFS.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.v2.hs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+/**
+ * Test service initialization of HistoryFileManager when
+ * HDFS is not running normally (either in start phase or
+ * in safe mode).
+ */
+public class TestHistoryFileManagerInitWithNonRunningDFS {
+ private static final String CLUSTER_BASE_DIR =
+ MiniDFSCluster.getBaseDirectory();
+
+ /**
+ * Verify if JHS keeps retrying to connect to HDFS, if the name node is
+ * in safe mode, when it creates history directories during service
+ * initialization. The expected behavior of JHS is to keep retrying for
+ * a time limit as specified by
+ * JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME, and give up by throwing
+ * a YarnRuntimeException with a time out message.
+ */
+ @Test
+ public void testKeepRetryingWhileNameNodeInSafeMode() throws Exception {
+ Configuration conf = new Configuration();
+ // set maximum wait time for JHS to wait for HDFS NameNode to start running
+ final long maxJhsWaitTime = 500;
+ conf.setLong(JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME, maxJhsWaitTime);
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, CLUSTER_BASE_DIR);
+
+ MiniDFSCluster dfsCluster = new MiniDFSCluster.Builder(conf).build();
+ try {
+ // set up a cluster with its name node in safe mode
+ dfsCluster.getFileSystem().setSafeMode(
+ HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ Assert.assertTrue(dfsCluster.getFileSystem().isInSafeMode());
+
+ HistoryFileManager hfm = new HistoryFileManager();
+ hfm.serviceInit(conf);
+ Assert.fail("History File Manager did not retry to connect to name node");
+ } catch (YarnRuntimeException yex) {
+ String expectedExceptionMsg = "Timed out '" + maxJhsWaitTime +
+ "ms' waiting for FileSystem to become available";
+ Assert.assertEquals("Unexpected reconnect timeout exception message",
+ expectedExceptionMsg, yex.getMessage());
+ } finally {
+ dfsCluster.shutdown(true);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HADOOP-13138. Unable to append to
a SequenceFile with Compression.NONE. Contributed by Vinayakumar B.
Posted by ar...@apache.org.
HADOOP-13138. Unable to append to a SequenceFile with Compression.NONE. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bad85f3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bad85f3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bad85f3e
Branch: refs/heads/HDFS-7240
Commit: bad85f3e3a53b5fb692c77df6c11dcf968ad7e62
Parents: cf552aa
Author: Chris Nauroth <cn...@apache.org>
Authored: Wed May 18 09:32:00 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Wed May 18 09:39:43 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/io/SequenceFile.java | 7 +-
.../hadoop/io/TestSequenceFileAppend.java | 67 ++++++++++++++++++++
2 files changed, 72 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad85f3e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
index ed57eee..0f3f317 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java
@@ -1115,9 +1115,12 @@ public class SequenceFile {
CompressionOption readerCompressionOption = new CompressionOption(
reader.getCompressionType(), reader.getCompressionCodec());
+ // Codec comparison will be ignored if the compression is NONE
if (readerCompressionOption.value != compressionTypeOption.value
- || !readerCompressionOption.codec.getClass().getName()
- .equals(compressionTypeOption.codec.getClass().getName())) {
+ || (readerCompressionOption.value != CompressionType.NONE
+ && readerCompressionOption.codec
+ .getClass() != compressionTypeOption.codec
+ .getClass())) {
throw new IllegalArgumentException(
"Compression option provided does not match the file");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bad85f3e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
index 9e49c9e..b31c809 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
@@ -249,6 +249,73 @@ public class TestSequenceFileAppend {
}
@Test(timeout = 30000)
+ public void testAppendNoneCompression() throws Exception {
+ Path file = new Path(ROOT_PATH, "testseqappendnonecompr.seq");
+ fs.delete(file, true);
+
+ Option compressOption = Writer.compression(CompressionType.NONE);
+ Writer writer =
+ SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+ SequenceFile.Writer.keyClass(Long.class),
+ SequenceFile.Writer.valueClass(String.class), compressOption);
+
+ writer.append(1L, "one");
+ writer.append(2L, "two");
+ writer.close();
+
+ verify2Values(file);
+
+ writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+ SequenceFile.Writer.keyClass(Long.class),
+ SequenceFile.Writer.valueClass(String.class),
+ SequenceFile.Writer.appendIfExists(true), compressOption);
+
+ writer.append(3L, "three");
+ writer.append(4L, "four");
+ writer.close();
+
+ verifyAll4Values(file);
+
+ // Verify failure if the compression details are different or not Provided
+ try {
+ writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+ SequenceFile.Writer.keyClass(Long.class),
+ SequenceFile.Writer.valueClass(String.class),
+ SequenceFile.Writer.appendIfExists(true));
+ writer.close();
+ fail("Expected IllegalArgumentException for compression options");
+ } catch (IllegalArgumentException iae) {
+ // Expected exception. Ignore it
+ }
+
+ // Verify failure if the compression details are different
+ try {
+ Option wrongCompressOption =
+ Writer.compression(CompressionType.RECORD, new GzipCodec());
+
+ writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+ SequenceFile.Writer.keyClass(Long.class),
+ SequenceFile.Writer.valueClass(String.class),
+ SequenceFile.Writer.appendIfExists(true), wrongCompressOption);
+ writer.close();
+ fail("Expected IllegalArgumentException for compression options");
+ } catch (IllegalArgumentException iae) {
+ // Expected exception. Ignore it
+ }
+
+ // Codec should be ignored
+ Option noneWithCodec =
+ Writer.compression(CompressionType.NONE, new DefaultCodec());
+
+ writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file),
+ SequenceFile.Writer.keyClass(Long.class),
+ SequenceFile.Writer.valueClass(String.class),
+ SequenceFile.Writer.appendIfExists(true), noneWithCodec);
+ writer.close();
+ fs.deleteOnExit(file);
+ }
+
+ @Test(timeout = 30000)
public void testAppendSort() throws Exception {
GenericTestUtils.assumeInNativeProfile();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: MAPREDUCE-6698. Increase timeout on
TestUnnecessaryBlockingOnHistoryFileInfo.testTwoThreadsQueryingDifferentJobOfSameUser.
Contributed by Haibo Chen
Posted by ar...@apache.org.
MAPREDUCE-6698. Increase timeout on TestUnnecessaryBlockingOnHistoryFileInfo.testTwoThreadsQueryingDifferentJobOfSameUser. Contributed by Haibo Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34fddd1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34fddd1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34fddd1e
Branch: refs/heads/HDFS-7240
Commit: 34fddd1e912dcd9f898e500b8df11295ee49c0d8
Parents: 12fa4ec
Author: Jason Lowe <jl...@apache.org>
Authored: Tue May 17 15:01:33 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue May 17 15:01:33 2016 +0000
----------------------------------------------------------------------
.../mapreduce/v2/hs/TestUnnecessaryBlockingOnHistoryFileInfo.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34fddd1e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestUnnecessaryBlockingOnHistoryFileInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestUnnecessaryBlockingOnHistoryFileInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestUnnecessaryBlockingOnHistoryFileInfo.java
index 06045b5..afe2e36 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestUnnecessaryBlockingOnHistoryFileInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestUnnecessaryBlockingOnHistoryFileInfo.java
@@ -83,7 +83,7 @@ public class TestUnnecessaryBlockingOnHistoryFileInfo {
* blocked by the other while the other thread is holding the lock on its
* associated job files and hanging up parsing the files.
*/
- @Test(timeout = 20000)
+ @Test(timeout = 600000)
public void testTwoThreadsQueryingDifferentJobOfSameUser()
throws InterruptedException, IOException {
final Configuration config = new Configuration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDFS-9732,
Improve DelegationTokenIdentifier.toString() for better logging.
Contributed by Yongjun Zhang.
Posted by ar...@apache.org.
HDFS-9732, Improve DelegationTokenIdentifier.toString() for better logging. Contributed by Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e24fe264
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e24fe264
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e24fe264
Branch: refs/heads/HDFS-7240
Commit: e24fe2641b4117601105fa097c8848d82b96b74c
Parents: 1356cbe
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Tue May 17 09:53:27 2016 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Tue May 17 14:03:08 2016 -0700
----------------------------------------------------------------------
.../AbstractDelegationTokenIdentifier.java | 33 +++++++++++++++++---
.../delegation/DelegationTokenIdentifier.java | 27 ++++++++++++++--
.../hdfs/tools/DelegationTokenFetcher.java | 32 ++++++++++++++-----
.../hdfs/security/TestDelegationToken.java | 2 +-
.../hdfs/tools/TestDelegationTokenFetcher.java | 17 ++++++++++
5 files changed, 96 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24fe264/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
index 5a9d076..0884cfd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
@@ -232,10 +232,35 @@ extends TokenIdentifier {
public String toString() {
StringBuilder buffer = new StringBuilder();
buffer
- .append("owner=" + owner + ", renewer=" + renewer + ", realUser="
- + realUser + ", issueDate=" + issueDate + ", maxDate=" + maxDate
- + ", sequenceNumber=" + sequenceNumber + ", masterKeyId="
- + masterKeyId);
+ .append(getKind())
+ .append(" owner=").append(owner)
+ .append(", renewer=").append(renewer)
+ .append(", realUser=").append(realUser)
+ .append(", issueDate=").append(issueDate)
+ .append(", maxDate=").append(maxDate)
+ .append(", sequenceNumber=").append(sequenceNumber)
+ .append(", masterKeyId=").append(masterKeyId);
+ return buffer.toString();
+ }
+ /*
+ * A frozen version of toString() to be used to be backward compatible.
+ * When backward compatibility is not needed, use toString(), which provides
+ * more info and is supposed to evolve, see HDFS-9732.
+ * Don't change this method except for major revisions.
+ *
+ * NOTE:
+ * Currently this method is used by CLI for backward compatibility.
+ */
+ public String toStringStable() {
+ StringBuilder buffer = new StringBuilder();
+ buffer
+ .append("owner=").append(owner)
+ .append(", renewer=").append(renewer)
+ .append(", realUser=").append(realUser)
+ .append(", issueDate=").append(issueDate)
+ .append(", maxDate=").append(maxDate)
+ .append(", sequenceNumber=").append(sequenceNumber)
+ .append(", masterKeyId=").append(masterKeyId);
return buffer.toString();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24fe264/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
index 62d5b9b..7bb0748 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java
@@ -60,9 +60,30 @@ public class DelegationTokenIdentifier
@Override
public String toString() {
- return getKind() + " token " + getSequenceNumber()
- + " for " + getUser().getShortUserName() + " with renewer " +
- getRenewer();
+ StringBuilder sbld = new StringBuilder();
+ sbld
+ .append("token for ").append(getUser().getShortUserName())
+ .append(": ").append(super.toString());
+ return sbld.toString();
+ }
+
+ /*
+ * A frozen version of toString() to be used to be backward compatible.
+ * When backward compatibility is not needed, use toString(), which provides
+ * more info and is supposed to evolve, see HDFS-9732.
+ * Don't change this method except for major revisions.
+ *
+ * NOTE:
+ * Currently this method is used by CLI for backward compatibility.
+ */
+ @Override
+ public String toStringStable() {
+ StringBuilder sbld = new StringBuilder();
+ sbld
+ .append(getKind()).append(" token ").append(getSequenceNumber())
+ .append(" for ").append(getUser().getShortUserName())
+ .append(" with renewer ").append(getRenewer());
+ return sbld.toString();
}
/** @return a string representation of the token */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24fe264/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index f24b50e..cf77804 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -62,7 +62,7 @@ public class DelegationTokenFetcher {
private static final String PRINT = "print";
private static final String RENEW = "renew";
private static final String RENEWER = "renewer";
-
+ private static final String VERBOSE = "verbose";
/**
* Command-line interface
*/
@@ -75,6 +75,7 @@ public class DelegationTokenFetcher {
.addOption(CANCEL, false, "cancel the token")
.addOption(RENEW, false, "renew the token")
.addOption(PRINT, false, "print the token")
+ .addOption(VERBOSE, false, "print verbose output")
.addOption(HELP_SHORT, HELP, false, "print out help information");
GenericOptionsParser parser = new GenericOptionsParser(conf,
@@ -88,6 +89,7 @@ public class DelegationTokenFetcher {
final boolean cancel = cmd.hasOption(CANCEL);
final boolean renew = cmd.hasOption(RENEW);
final boolean print = cmd.hasOption(PRINT);
+ final boolean verbose = cmd.hasOption(VERBOSE);
final boolean help = cmd.hasOption(HELP);
String[] remaining = parser.getRemainingArgs();
@@ -115,7 +117,7 @@ public class DelegationTokenFetcher {
@Override
public Object run() throws Exception {
if (print) {
- printTokens(conf, tokenFile);
+ printTokens(conf, tokenFile, verbose);
} else if (cancel) {
cancelTokens(conf, tokenFile);
} else if (renew) {
@@ -191,17 +193,32 @@ public class DelegationTokenFetcher {
}
}
- private static void printTokens(final Configuration conf,
- final Path tokenFile)
- throws IOException {
+ @VisibleForTesting
+ static String printTokensToString(
+ final Configuration conf,
+ final Path tokenFile,
+ final boolean verbose) throws IOException {
+ StringBuilder sbld = new StringBuilder();
+ final String nl = System.getProperty("line.separator");
DelegationTokenIdentifier id = new DelegationTokenSecretManager(0, 0, 0,
0, null).createIdentifier();
for (Token<?> token : readTokens(tokenFile, conf)) {
DataInputStream in = new DataInputStream(new ByteArrayInputStream(token
.getIdentifier()));
id.readFields(in);
- System.out.println("Token (" + id + ") for " + token.getService());
+ String idStr = (verbose? id.toString() : id.toStringStable());
+ sbld
+ .append("Token (").append(idStr)
+ .append(") for ").append(token.getService()).append(nl);
}
+ return sbld.toString();
+ }
+
+ // Be sure to call printTokensToString which is verified in unit test.
+ static void printTokens(final Configuration conf,
+ final Path tokenFile,
+ final boolean verbose) throws IOException {
+ System.out.print(printTokensToString(conf, tokenFile, verbose));
}
private static void printUsage(PrintStream err) {
@@ -216,7 +233,8 @@ public class DelegationTokenFetcher {
err.println(" --renew Renew the delegation token. " +
"Delegation " + "token must have been fetched using the --renewer" +
" <name> option.");
- err.println(" --print Print the delegation token");
+ err.println(" --print [--verbose] Print the delegation token, when " +
+ "--verbose is passed, print more information about the token");
err.println();
GenericOptionsParser.printGenericCommandUsage(err);
ExitUtil.terminate(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24fe264/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
index f4aae8f..0600536 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
@@ -322,6 +322,6 @@ public class TestDelegationToken {
"SomeUser"), new Text("JobTracker"), null);
Assert.assertEquals("HDFS_DELEGATION_TOKEN token 0" +
" for SomeUser with renewer JobTracker",
- dtId.toString());
+ dtId.toStringStable());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24fe264/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
index 80a1a6c..dbaae9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
@@ -48,8 +48,12 @@ import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestDelegationTokenFetcher {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestDelegationTokenFetcher.class);
private Configuration conf = new Configuration();
@@ -132,6 +136,19 @@ public class TestDelegationTokenFetcher {
Iterator<Token<?>> itr = creds.getAllTokens().iterator();
assertTrue("token not exist error", itr.hasNext());
assertNotNull("Token should be there without renewer", itr.next());
+
+ // Test compatibility of DelegationTokenFetcher.printTokensToString
+ String expectedNonVerbose = "Token (HDFS_DELEGATION_TOKEN token 1 for " +
+ System.getProperty("user.name") + " with renewer ) for";
+ String resNonVerbose =
+ DelegationTokenFetcher.printTokensToString(conf, p, false);
+ assertTrue("The non verbose output is expected to start with \""
+ + expectedNonVerbose +"\"",
+ resNonVerbose.startsWith(expectedNonVerbose));
+ LOG.info(resNonVerbose);
+ LOG.info(
+ DelegationTokenFetcher.printTokensToString(conf, p, true));
+
try {
// Without renewer renewal of token should fail.
DelegationTokenFetcher.renewTokens(conf, p);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HADOOP-13158
S3AFileSystem#toString might throw NullPointerException due to null
cannedACL. (Chris Nauroth via stevel)
Posted by ar...@apache.org.
HADOOP-13158 S3AFileSystem#toString might throw NullPointerException due to null cannedACL. (Chris Nauroth via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08ea07f1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08ea07f1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08ea07f1
Branch: refs/heads/HDFS-7240
Commit: 08ea07f1b8edbc38c99015c81a62ca127a247bf7
Parents: c69a649
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 17 13:18:51 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue May 17 13:19:29 2016 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08ea07f1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 3f9723d..44b8aae 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -1380,7 +1380,9 @@ public class S3AFileSystem extends FileSystem {
sb.append(", partSize=").append(partSize);
sb.append(", enableMultiObjectsDelete=").append(enableMultiObjectsDelete);
sb.append(", maxKeys=").append(maxKeys);
- sb.append(", cannedACL=").append(cannedACL.toString());
+ if (cannedACL != null) {
+ sb.append(", cannedACL=").append(cannedACL.toString());
+ }
sb.append(", readAhead=").append(readAhead);
sb.append(", blockSize=").append(getDefaultBlockSize());
sb.append(", multiPartThreshold=").append(multiPartThreshold);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HADOOP-13159. Fix potential NPE in
Metrics2 source for DecayRpcScheduler. Contributed by Xiaoyu Yao.
Posted by ar...@apache.org.
HADOOP-13159. Fix potential NPE in Metrics2 source for DecayRpcScheduler. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94784848
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94784848
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94784848
Branch: refs/heads/HDFS-7240
Commit: 94784848456a92a6502f3a3c0074e44fba4b19c9
Parents: 0c6726e
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue May 17 14:57:51 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue May 17 14:58:22 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/ipc/DecayRpcScheduler.java | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94784848/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
index b39bda2..3443d03 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
@@ -206,11 +206,6 @@ public class DecayRpcScheduler implements RpcScheduler,
this.backOffResponseTimeThresholds =
parseBackOffResponseTimeThreshold(ns, conf, numLevels);
- // Setup delay timer
- Timer timer = new Timer();
- DecayTask task = new DecayTask(this, timer);
- timer.scheduleAtFixedRate(task, decayPeriodMillis, decayPeriodMillis);
-
// Setup response time metrics
responseTimeTotalInCurrWindow = new AtomicLongArray(numLevels);
responseTimeCountInCurrWindow = new AtomicLongArray(numLevels);
@@ -223,6 +218,11 @@ public class DecayRpcScheduler implements RpcScheduler,
Preconditions.checkArgument(topUsersCount > 0,
"the number of top users for scheduler metrics must be at least 1");
+ // Setup delay timer
+ Timer timer = new Timer();
+ DecayTask task = new DecayTask(this, timer);
+ timer.scheduleAtFixedRate(task, decayPeriodMillis, decayPeriodMillis);
+
MetricsProxy prox = MetricsProxy.getInstance(ns, numLevels);
prox.setDelegate(this);
prox.registerMetrics2Source(ns);
@@ -821,9 +821,10 @@ public class DecayRpcScheduler implements RpcScheduler,
final int topCallerCount = 10;
TopN topNCallers = getTopCallers(topCallerCount);
Map<Object, Integer> decisions = scheduleCacheRef.get();
- for (int i=0; i < topNCallers.size(); i++) {
+ final int actualCallerCount = topNCallers.size();
+ for (int i = 0; i < actualCallerCount; i++) {
NameValuePair entry = topNCallers.poll();
- String topCaller = "Top." + (topCallerCount - i) + "." +
+ String topCaller = "Top." + (actualCallerCount - i) + "." +
"Caller(" + entry.getName() + ")";
String topCallerVolume = topCaller + ".Volume";
String topCallerPriority = topCaller + ".Priority";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-5069.
TestFifoScheduler.testResourceOverCommit race condition. Contributed by Eric
Badger.
Posted by ar...@apache.org.
YARN-5069. TestFifoScheduler.testResourceOverCommit race condition. Contributed by Eric Badger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1217c8f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1217c8f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1217c8f6
Branch: refs/heads/HDFS-7240
Commit: 1217c8f6b49e3afd8ca008ffddcf6615f0accfc5
Parents: 576e2d1
Author: Eric Payne <ep...@apache.org>
Authored: Mon May 16 20:27:07 2016 +0000
Committer: Eric Payne <ep...@apache.org>
Committed: Mon May 16 20:28:04 2016 +0000
----------------------------------------------------------------------
.../server/resourcemanager/scheduler/fifo/TestFifoScheduler.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1217c8f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 90b9969..02c60cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -1140,7 +1140,8 @@ public class TestFifoScheduler {
waitCount = 0;
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- if (report_nm1.getAvailableResource().getMemory() != 0) {
+ if (null != report_nm1 &&
+ report_nm1.getAvailableResource().getMemory() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HADOOP-13163 Reuse pre-computed
filestatus in Distcp-CopyMapper (Rajesh Balamohan via stevel)
Posted by ar...@apache.org.
HADOOP-13163 Reuse pre-computed filestatus in Distcp-CopyMapper (Rajesh Balamohan via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c69a6492
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c69a6492
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c69a6492
Branch: refs/heads/HDFS-7240
Commit: c69a649257a331da55c1a1bf61c819e289015a6b
Parents: b357930
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 17 12:59:59 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue May 17 13:00:18 2016 +0100
----------------------------------------------------------------------
.../java/org/apache/hadoop/tools/mapred/CopyMapper.java | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c69a6492/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
index 4db1d4e..4b73415 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
@@ -182,7 +182,7 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
return;
}
- FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target);
+ FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, targetStatus);
if (action == FileAction.SKIP) {
LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
+ " to " + target);
@@ -270,13 +270,7 @@ public class CopyMapper extends Mapper<Text, CopyListingFileStatus, Text, Text>
}
private FileAction checkUpdate(FileSystem sourceFS, FileStatus source,
- Path target) throws IOException {
- final FileStatus targetFileStatus;
- try {
- targetFileStatus = targetFS.getFileStatus(target);
- } catch (FileNotFoundException e) {
- return FileAction.OVERWRITE;
- }
+ Path target, FileStatus targetFileStatus) throws IOException {
if (targetFileStatus != null && !overWrite) {
if (canSkip(sourceFS, source, targetFileStatus)) {
return FileAction.SKIP;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: MAPREDUCE-6693.
ArrayIndexOutOfBoundsException occurs when the length of the job name is
equal to mapreduce.jobhistory.jobname.limit. Contributed by Ajith S.
Posted by ar...@apache.org.
MAPREDUCE-6693. ArrayIndexOutOfBoundsException occurs when the length of the job name is equal to mapreduce.jobhistory.jobname.limit. Contributed by Ajith S.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3579305
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3579305
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3579305
Branch: refs/heads/HDFS-7240
Commit: b3579305268bffc44a8041c5b75f15f01ebb9ce4
Parents: 9fe5828
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue May 17 20:19:01 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue May 17 20:19:01 2016 +0900
----------------------------------------------------------------------
.../v2/jobhistory/FileNameIndexUtils.java | 2 +-
.../v2/jobhistory/TestFileNameIndexUtils.java | 25 ++++++++++++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3579305/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
index 284fe80..aac3cb2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
@@ -311,7 +311,7 @@ public class FileNameIndexUtils {
String encodedString, int limitLength) {
assert(limitLength >= 0) : "limitLength should be positive integer";
- if (encodedString.length() < limitLength) {
+ if (encodedString.length() <= limitLength) {
return encodedString;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3579305/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
index 6fa7dbc..f9322d8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
@@ -481,4 +481,29 @@ public class TestFileNameIndexUtils {
Assert.assertNull("Queue name incorrect after decoding old history file",
info.getQueueName());
}
+
+ @Test
+ public void testTrimJobNameEqualsLimitLength() throws IOException {
+ int jobNameTrimLength = 9;
+ JobIndexInfo info = new JobIndexInfo();
+ JobID oldJobId = JobID.forName(JOB_ID);
+ JobId jobId = TypeConverter.toYarn(oldJobId);
+ info.setJobId(jobId);
+ info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
+ info.setUser(USER_NAME);
+ info.setJobName(JOB_NAME);
+ info.setFinishTime(Long.parseLong(FINISH_TIME));
+ info.setNumMaps(Integer.parseInt(NUM_MAPS));
+ info.setNumReduces(Integer.parseInt(NUM_REDUCES));
+ info.setJobStatus(JOB_STATUS);
+ info.setQueueName(QUEUE_NAME);
+ info.setJobStartTime(Long.parseLong(JOB_START_TIME));
+
+ String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info,
+ jobNameTrimLength);
+ JobIndexInfo parsedInfo = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
+
+ Assert.assertEquals("Job name did not get trimmed correctly", info
+ .getJobName().substring(0, jobNameTrimLength), parsedInfo.getJobName());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-4325. Nodemanager log
handlers fail to send finished/failed events in some cases. Contributed by
Junping Du
Posted by ar...@apache.org.
YARN-4325. Nodemanager log handlers fail to send finished/failed events in some cases. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81effb7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81effb7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81effb7d
Branch: refs/heads/HDFS-7240
Commit: 81effb7dcde2b31423438d6f1b8b8204d4ca05b3
Parents: ad94411
Author: Jason Lowe <jl...@apache.org>
Authored: Mon May 16 15:40:23 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon May 16 15:40:23 2016 +0000
----------------------------------------------------------------------
.../application/ApplicationImpl.java | 8 +-
.../logaggregation/AppLogAggregatorImpl.java | 4 +
.../logaggregation/LogAggregationService.java | 5 +-
.../loghandler/NonAggregatingLogHandler.java | 4 +
.../TestContainerManagerRecovery.java | 84 ++++++++++++++++++++
.../TestNonAggregatingLogHandler.java | 58 +++++++++++++-
6 files changed, 155 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81effb7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index fbc8453..efa258a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -207,18 +207,18 @@ public class ApplicationImpl implements Application {
ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED,
ApplicationEventType.APPLICATION_INITED,
ApplicationEventType.FINISH_APPLICATION))
-
+
// Transitions from FINISHED state
.addTransition(ApplicationState.FINISHED,
ApplicationState.FINISHED,
- ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED,
+ EnumSet.of(
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED),
new AppLogsAggregatedTransition())
.addTransition(ApplicationState.FINISHED, ApplicationState.FINISHED,
EnumSet.of(
ApplicationEventType.APPLICATION_LOG_HANDLING_INITED,
- ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED,
ApplicationEventType.FINISH_APPLICATION))
-
// create the topology tables
.installTopology();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81effb7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index fed4a3b..32b0934 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -501,6 +501,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
}
}
+ @SuppressWarnings("unchecked")
@Override
public void run() {
try {
@@ -513,6 +514,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
} finally {
if (!this.appAggregationFinished.get()) {
LOG.warn("Aggregation did not complete for application " + appId);
+ this.dispatcher.getEventHandler().handle(
+ new ApplicationEvent(this.appId,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
}
this.appAggregationFinished.set(true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81effb7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
index d46f7a3..189e7ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
@@ -416,7 +416,6 @@ public class LogAggregationService extends AbstractService implements
// A container is complete. Put this containers' logs up for aggregation if
// this containers' logs are needed.
-
AppLogAggregator aggregator = this.appLogAggregators.get(
containerId.getApplicationAttemptId().getApplicationId());
if (aggregator == null) {
@@ -436,6 +435,7 @@ public class LogAggregationService extends AbstractService implements
new ContainerLogContext(containerId, containerType, exitCode));
}
+ @SuppressWarnings("unchecked")
private void stopApp(ApplicationId appId) {
// App is complete. Finish up any containers' pending log aggregation and
@@ -445,6 +445,9 @@ public class LogAggregationService extends AbstractService implements
if (aggregator == null) {
LOG.warn("Log aggregation is not initialized for " + appId
+ ", did it fail to start?");
+ this.dispatcher.getEventHandler().handle(
+ new ApplicationEvent(appId,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
return;
}
aggregator.finishLogAggregation();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81effb7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
index d42a4e7..2901743 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
@@ -171,6 +171,10 @@ public class NonAggregatingLogHandler extends AbstractService implements
String user = appOwners.remove(appId);
if (user == null) {
LOG.error("Unable to locate user for " + appId);
+ // send LOG_HANDLING_FAILED out
+ NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle(
+ new ApplicationEvent(appId,
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
break;
}
LogDeleterRunnable logDeleter = new LogDeleterRunnable(user, appId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81effb7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index ccdfd64..61477a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -295,6 +295,90 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
}
@Test
+ public void testNMRecoveryForAppFinishedWithLogAggregationFailure()
+ throws Exception {
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
+
+ NMStateStoreService stateStore = new NMMemoryStateStoreService();
+ stateStore.init(conf);
+ stateStore.start();
+ Context context = createContext(conf, stateStore);
+ ContainerManagerImpl cm = createContainerManager(context);
+ cm.init(conf);
+ cm.start();
+
+ // add an application by starting a container
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId attemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId cid = ContainerId.newContainerId(attemptId, 1);
+ Map<String, LocalResource> localResources = Collections.emptyMap();
+ Map<String, String> containerEnv = Collections.emptyMap();
+ List<String> containerCmds = Collections.emptyList();
+ Map<String, ByteBuffer> serviceData = Collections.emptyMap();
+
+ ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
+ localResources, containerEnv, containerCmds, serviceData,
+ null, null);
+
+ StartContainersResponse startResponse = startContainer(context, cm, cid,
+ clc, null);
+ assertTrue(startResponse.getFailedRequests().isEmpty());
+ assertEquals(1, context.getApplications().size());
+ Application app = context.getApplications().get(appId);
+ assertNotNull(app);
+ waitForAppState(app, ApplicationState.INITING);
+
+ // simulate application completion
+ List<ApplicationId> finishedApps = new ArrayList<ApplicationId>();
+ finishedApps.add(appId);
+ cm.handle(new CMgrCompletedAppsEvent(finishedApps,
+ CMgrCompletedAppsEvent.Reason.BY_RESOURCEMANAGER));
+ waitForAppState(app, ApplicationState.APPLICATION_RESOURCES_CLEANINGUP);
+
+ app.handle(new ApplicationEvent(app.getAppId(),
+ ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP));
+ assertEquals(app.getApplicationState(), ApplicationState.FINISHED);
+ // application is still in NM context.
+ assertEquals(1, context.getApplications().size());
+
+ // restart and verify app is still there and marked as finished.
+ cm.stop();
+ context = createContext(conf, stateStore);
+ cm = createContainerManager(context);
+ cm.init(conf);
+ cm.start();
+ assertEquals(1, context.getApplications().size());
+ app = context.getApplications().get(appId);
+ assertNotNull(app);
+
+ // no longer saving FINISH_APP event in NM stateStore,
+ // simulate by resending FINISH_APP event
+ cm.handle(new CMgrCompletedAppsEvent(finishedApps,
+ CMgrCompletedAppsEvent.Reason.BY_RESOURCEMANAGER));
+ waitForAppState(app, ApplicationState.APPLICATION_RESOURCES_CLEANINGUP);
+ // TODO need to figure out why additional APPLICATION_RESOURCES_CLEANEDUP
+ // is needed.
+ app.handle(new ApplicationEvent(app.getAppId(),
+ ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP));
+ assertEquals(app.getApplicationState(), ApplicationState.FINISHED);
+
+ // simulate log aggregation failed.
+ app.handle(new ApplicationEvent(app.getAppId(),
+ ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
+
+ // restart and verify app is no longer present after recovery
+ cm.stop();
+ context = createContext(conf, stateStore);
+ cm = createContainerManager(context);
+ cm.init(conf);
+ cm.start();
+ assertTrue(context.getApplications().isEmpty());
+ cm.stop();
+ }
+
+ @Test
public void testContainerResizeRecovery() throws Exception {
conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81effb7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
index 46d06da..ec3757e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
@@ -84,7 +87,7 @@ public class TestNonAggregatingLogHandler {
DeletionService mockDelService;
Configuration conf;
DrainDispatcher dispatcher;
- EventHandler<ApplicationEvent> appEventHandler;
+ private ApplicationEventHandler appEventHandler;
String user = "testuser";
ApplicationId appId;
ApplicationAttemptId appAttemptId;
@@ -97,7 +100,7 @@ public class TestNonAggregatingLogHandler {
mockDelService = mock(DeletionService.class);
conf = new YarnConfiguration();
dispatcher = createDispatcher(conf);
- appEventHandler = mock(EventHandler.class);
+ appEventHandler = new ApplicationEventHandler();
dispatcher.register(ApplicationEventType.class, appEventHandler);
appId = BuilderUtils.newApplicationId(1234, 1);
appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
@@ -345,6 +348,9 @@ public class TestNonAggregatingLogHandler {
dirsHandler.init(conf);
+ appEventHandler.resetLogHandlingEvent();
+ assertFalse(appEventHandler.receiveLogHandlingFinishEvent());
+
NMStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.init(conf);
stateStore.start();
@@ -377,8 +383,21 @@ public class TestNonAggregatingLogHandler {
logHandler.start();
verify(logHandler.mockSched, never()).schedule(any(Runnable.class),
anyLong(), any(TimeUnit.class));
+
+ // wait events get drained.
+ this.dispatcher.await();
+ assertTrue(appEventHandler.receiveLogHandlingFinishEvent());
+
+ appEventHandler.resetLogHandlingEvent();
+ assertFalse(appEventHandler.receiveLogHandlingFailedEvent());
+ // send an app finish event against a removed app
+ logHandler.handle(new LogHandlerAppFinishedEvent(appId));
+ this.dispatcher.await();
+ // verify to receive a log failed event.
+ assertTrue(appEventHandler.receiveLogHandlingFailedEvent());
+ assertFalse(appEventHandler.receiveLogHandlingFinishEvent());
logHandler.close();
- }
+ }
/**
* Function to run a log handler with directories failing the getFileStatus
@@ -536,4 +555,37 @@ public class TestNonAggregatingLogHandler {
}
return dirs;
}
+
+ class ApplicationEventHandler implements EventHandler<ApplicationEvent> {
+
+ private boolean logHandlingFinished = false;
+ private boolean logHandlingFailed = false;
+
+ @Override
+ public void handle(ApplicationEvent event) {
+ switch (event.getType()) {
+ case APPLICATION_LOG_HANDLING_FINISHED:
+ logHandlingFinished = true;
+ break;
+ case APPLICATION_LOG_HANDLING_FAILED:
+ logHandlingFailed = true;
+ default:
+ // do nothing.
+ }
+ }
+
+ public boolean receiveLogHandlingFinishEvent() {
+ return logHandlingFinished;
+ }
+
+ public boolean receiveLogHandlingFailedEvent() {
+ return logHandlingFailed;
+ }
+
+ public void resetLogHandlingEvent() {
+ logHandlingFinished = false;
+ logHandlingFailed = false;
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HADOOP-12930. Dynamic subcommands
for hadoop shell scripts (aw)
Posted by ar...@apache.org.
HADOOP-12930. Dynamic subcommands for hadoop shell scripts (aw)
This commit contains the following JIRA issues:
HADOOP-12931. bin/hadoop work for dynamic subcommands
HADOOP-12932. bin/yarn work for dynamic subcommands
HADOOP-12933. bin/hdfs work for dynamic subcommands
HADOOP-12934. bin/mapred work for dynamic subcommands
HADOOP-12935. API documentation for dynamic subcommands
HADOOP-12936. modify hadoop-tools to take advantage of dynamic subcommands
HADOOP-13086. enable daemonization of dynamic commands
HADOOP-13087. env var doc update for dynamic commands
HADOOP-13088. fix shellprofiles in hadoop-tools to allow replacement
HADOOP-13089. hadoop distcp adds client opts twice when dynamic
HADOOP-13094. hadoop-common unit tests for dynamic commands
HADOOP-13095. hadoop-hdfs unit tests for dynamic commands
HADOOP-13107. clean up how rumen is executed
HADOOP-13108. dynamic subcommands need a way to manipulate arguments
HADOOP-13110. add a streaming subcommand to mapred
HADOOP-13111. convert hadoop gridmix to be dynamic
HADOOP-13115. dynamic subcommand docs should talk about exit vs. continue program flow
HADOOP-13117. clarify daemonization and security vars for dynamic commands
HADOOP-13120. add a --debug message when dynamic commands have been used
HADOOP-13121. rename sub-project shellprofiles to match the rest of Hadoop
HADOOP-13129. fix typo in dynamic subcommand docs
HADOOP-13151. Underscores should be escaped in dynamic subcommands document
HADOOP-13153. fix typo in debug statement for dynamic subcommands
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/730bc746
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/730bc746
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/730bc746
Branch: refs/heads/HDFS-7240
Commit: 730bc746f9ac6e045e94dc2bc622b16de0159b4b
Parents: 6a6e74a
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Mar 28 09:00:07 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon May 16 17:54:45 2016 -0700
----------------------------------------------------------------------
.../main/resources/assemblies/hadoop-tools.xml | 56 +++
.../hadoop-common/src/main/bin/hadoop | 323 ++++++++------
.../src/site/markdown/UnixShellGuide.md | 87 +++-
.../scripts/hadoop-functions_test_helper.bash | 4 +-
.../src/test/scripts/hadoop_subcommands.bats | 78 ++++
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 38 +-
.../hadoop-hdfs/src/main/bin/hdfs | 430 ++++++++++---------
.../src/main/shellprofile.d/hadoop-hdfs.sh | 36 ++
.../hadoop-hdfs/src/main/shellprofile.d/hdfs.sh | 36 --
.../scripts/hdfs-functions_test_helper.bash | 58 +++
.../src/test/scripts/hdfs_subcommands.bats | 78 ++++
.../hadoop-hdfs/src/test/scripts/run-bats.sh | 43 ++
hadoop-mapreduce-project/bin/mapred | 230 +++++-----
.../shellprofile.d/hadoop-mapreduce.sh | 41 ++
.../shellprofile.d/mapreduce.sh | 41 --
.../main/shellprofile.d/hadoop-archive-logs.sh | 39 ++
.../src/main/shellprofile.d/hadoop-archives.sh | 58 +++
.../src/main/shellprofile.d/hadoop-distcp.sh | 62 +++
.../src/main/shellprofile.d/hadoop-extras.sh | 39 ++
.../src/main/shellprofile.d/hadoop-gridmix.sh | 36 ++
.../src/main/shellprofile.d/hadoop-rumen.sh | 58 +++
.../hadoop-rumen/src/site/markdown/Rumen.md.vm | 40 +-
.../apache/hadoop/streaming/DumpTypedBytes.java | 3 +-
.../hadoop/streaming/HadoopStreaming.java | 3 +-
.../apache/hadoop/streaming/LoadTypedBytes.java | 3 +-
.../src/main/shellprofile.d/hadoop-streaming.sh | 55 +++
.../src/site/markdown/HadoopStreaming.md.vm | 30 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 394 +++++++++--------
.../hadoop-yarn/shellprofile.d/hadoop-yarn.sh | 62 +++
.../hadoop-yarn/shellprofile.d/yarn.sh | 62 ---
30 files changed, 1721 insertions(+), 802 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index f8ba48e..bc9548b 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -24,6 +24,38 @@
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
+ <directory>../hadoop-archive-logs/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../hadoop-archives/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../hadoop-distcp/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>../hadoop-extras/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
<includes>
<include>*.hh</include>
@@ -94,6 +126,14 @@
</includes>
</fileSet>
<fileSet>
+ <directory>../hadoop-gridmix/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>../hadoop-rumen/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
@@ -101,6 +141,14 @@
</includes>
</fileSet>
<fileSet>
+ <directory>../hadoop-rumen/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>../hadoop-streaming/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
@@ -108,6 +156,14 @@
</includes>
</fileSet>
<fileSet>
+ <directory>../hadoop-streaming/src/main/shellprofile.d</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <outputDirectory>/libexec/shellprofile.d</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>../hadoop-sls/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index fccb9f8..6cf872c 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -16,7 +16,12 @@
# limitations under the License.
MYNAME="${BASH_SOURCE-$0}"
+HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
+## @description build up the hadoop command's usage text.
+## @audience public
+## @stability stable
+## @replaceable no
function hadoop_usage
{
hadoop_add_option "buildpaths" "attempt to add class files from build tree"
@@ -25,25 +30,141 @@ function hadoop_usage
hadoop_add_option "hosts filename" "list of hosts to use in slave mode"
hadoop_add_option "slaves" "turn on slave mode"
- hadoop_add_subcommand "archive" "create a Hadoop archive"
hadoop_add_subcommand "checknative" "check native Hadoop and compression libraries availability"
hadoop_add_subcommand "classpath" "prints the class path needed to get the Hadoop jar and the required libraries"
hadoop_add_subcommand "conftest" "validate configuration XML files"
hadoop_add_subcommand "credential" "interact with credential providers"
hadoop_add_subcommand "daemonlog" "get/set the log level for each daemon"
- hadoop_add_subcommand "distch" "distributed metadata changer"
- hadoop_add_subcommand "distcp" "copy file or directories recursively"
hadoop_add_subcommand "dtutil" "operations related to delegation tokens"
hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
hadoop_add_subcommand "fs" "run a generic filesystem user client"
- hadoop_add_subcommand "gridmix" "submit a mix of synthetic job, modeling a profiled from production load"
hadoop_add_subcommand "jar <jar>" "run a jar file. NOTE: please use \"yarn jar\" to launch YARN applications, not this command."
hadoop_add_subcommand "jnipath" "prints the java.library.path"
hadoop_add_subcommand "kerbname" "show auth_to_local principal conversion"
hadoop_add_subcommand "key" "manage keys via the KeyProvider"
hadoop_add_subcommand "trace" "view and modify Hadoop tracing settings"
hadoop_add_subcommand "version" "print the version"
- hadoop_generate_usage "${MYNAME}" true
+ hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
+}
+
+## @description Default command handler for hadoop command
+## @audience public
+## @stability stable
+## @replaceable no
+## @param CLI arguments
+function hadoopcmd_case
+{
+ subcmd=$1
+ shift
+
+ case ${subcmd} in
+ balancer|datanode|dfs|dfsadmin|dfsgroups| \
+ namenode|secondarynamenode|fsck|fetchdt|oiv| \
+ portmap|nfs3)
+ hadoop_error "WARNING: Use of this script to execute ${subcmd} is deprecated."
+ subcmd=${subcmd/dfsgroups/groups}
+ hadoop_error "WARNING: Attempting to execute replacement \"hdfs ${subcmd}\" instead."
+ hadoop_error ""
+ #try to locate hdfs and if present, delegate to it.
+ if [[ -f "${HADOOP_HDFS_HOME}/bin/hdfs" ]]; then
+ # shellcheck disable=SC2086
+ exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
+ --config "${HADOOP_CONF_DIR}" "${subcmd}" "$@"
+ elif [[ -f "${HADOOP_HOME}/bin/hdfs" ]]; then
+ # shellcheck disable=SC2086
+ exec "${HADOOP_HOME}/bin/hdfs" \
+ --config "${HADOOP_CONF_DIR}" "${subcmd}" "$@"
+ else
+ hadoop_error "HADOOP_HDFS_HOME not found!"
+ exit 1
+ fi
+ ;;
+
+ #mapred commands for backwards compatibility
+ pipes|job|queue|mrgroups|mradmin|jobtracker|tasktracker)
+ hadoop_error "WARNING: Use of this script to execute ${subcmd} is deprecated."
+ subcmd=${subcmd/mrgroups/groups}
+ hadoop_error "WARNING: Attempting to execute replacement \"mapred ${subcmd}\" instead."
+ hadoop_error ""
+ #try to locate mapred and if present, delegate to it.
+ if [[ -f "${HADOOP_MAPRED_HOME}/bin/mapred" ]]; then
+ exec "${HADOOP_MAPRED_HOME}/bin/mapred" \
+ --config "${HADOOP_CONF_DIR}" "${subcmd}" "$@"
+ elif [[ -f "${HADOOP_HOME}/bin/mapred" ]]; then
+ exec "${HADOOP_HOME}/bin/mapred" \
+ --config "${HADOOP_CONF_DIR}" "${subcmd}" "$@"
+ else
+ hadoop_error "HADOOP_MAPRED_HOME not found!"
+ exit 1
+ fi
+ ;;
+ checknative)
+ HADOOP_CLASSNAME=org.apache.hadoop.util.NativeLibraryChecker
+ ;;
+ classpath)
+ hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
+ ;;
+ conftest)
+ HADOOP_CLASSNAME=org.apache.hadoop.util.ConfTest
+ ;;
+ credential)
+ HADOOP_CLASSNAME=org.apache.hadoop.security.alias.CredentialShell
+ ;;
+ daemonlog)
+ HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
+ ;;
+ dtutil)
+ HADOOP_CLASSNAME=org.apache.hadoop.security.token.DtUtilShell
+ ;;
+ envvars)
+ echo "JAVA_HOME='${JAVA_HOME}'"
+ echo "HADOOP_COMMON_HOME='${HADOOP_COMMON_HOME}'"
+ echo "HADOOP_COMMON_DIR='${HADOOP_COMMON_DIR}'"
+ echo "HADOOP_COMMON_LIB_JARS_DIR='${HADOOP_COMMON_LIB_JARS_DIR}'"
+ echo "HADOOP_COMMON_LIB_NATIVE_DIR='${HADOOP_COMMON_LIB_NATIVE_DIR}'"
+ echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
+ echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+ echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+ echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
+ exit 0
+ ;;
+ fs)
+ HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
+ ;;
+ jar)
+ if [[ -n "${YARN_OPTS}" ]] || [[ -n "${YARN_CLIENT_OPTS}" ]]; then
+ hadoop_error "WARNING: Use \"yarn jar\" to launch YARN applications."
+ fi
+ HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
+ ;;
+ jnipath)
+ hadoop_finalize
+ echo "${JAVA_LIBRARY_PATH}"
+ exit 0
+ ;;
+ kerbname)
+ HADOOP_CLASSNAME=org.apache.hadoop.security.HadoopKerberosName
+ ;;
+ key)
+ HADOOP_CLASSNAME=org.apache.hadoop.crypto.key.KeyShell
+ ;;
+ trace)
+ HADOOP_CLASSNAME=org.apache.hadoop.tracing.TraceAdmin
+ ;;
+ version)
+ HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
+ ;;
+ *)
+ HADOOP_CLASSNAME="${subcmd}"
+ if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
+ hadoop_exit_with_usage 1
+ fi
+ ;;
+ esac
+
+ # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
}
# This script runs the hadoop core commands.
@@ -70,142 +191,76 @@ if [ $# = 0 ]; then
hadoop_exit_with_usage 1
fi
-COMMAND=$1
+HADOOP_SUBCMD=$1
shift
-case ${COMMAND} in
- balancer|datanode|dfs|dfsadmin|dfsgroups| \
- namenode|secondarynamenode|fsck|fetchdt|oiv| \
- portmap|nfs3)
- hadoop_error "WARNING: Use of this script to execute ${COMMAND} is deprecated."
- COMMAND=${COMMAND/dfsgroups/groups}
- hadoop_error "WARNING: Attempting to execute replacement \"hdfs ${COMMAND}\" instead."
- hadoop_error ""
- #try to locate hdfs and if present, delegate to it.
- if [[ -f "${HADOOP_HDFS_HOME}/bin/hdfs" ]]; then
- # shellcheck disable=SC2086
- exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
- --config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
- elif [[ -f "${HADOOP_HOME}/bin/hdfs" ]]; then
- # shellcheck disable=SC2086
- exec "${HADOOP_HOME}/bin/hdfs" \
- --config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
- else
- hadoop_error "HADOOP_HDFS_HOME not found!"
- exit 1
- fi
- ;;
-
- #mapred commands for backwards compatibility
- pipes|job|queue|mrgroups|mradmin|jobtracker|tasktracker)
- hadoop_error "WARNING: Use of this script to execute ${COMMAND} is deprecated."
- COMMAND=${COMMAND/mrgroups/groups}
- hadoop_error "WARNING: Attempting to execute replacement \"mapred ${COMMAND}\" instead."
- hadoop_error ""
- #try to locate mapred and if present, delegate to it.
- if [[ -f "${HADOOP_MAPRED_HOME}/bin/mapred" ]]; then
- exec "${HADOOP_MAPRED_HOME}/bin/mapred" \
- --config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
- elif [[ -f "${HADOOP_HOME}/bin/mapred" ]]; then
- exec "${HADOOP_HOME}/bin/mapred" \
- --config "${HADOOP_CONF_DIR}" "${COMMAND}" "$@"
- else
- hadoop_error "HADOOP_MAPRED_HOME not found!"
- exit 1
- fi
- ;;
- archive)
- CLASS=org.apache.hadoop.tools.HadoopArchives
- hadoop_add_to_classpath_tools hadoop-archives
- ;;
- checknative)
- CLASS=org.apache.hadoop.util.NativeLibraryChecker
- ;;
- classpath)
- hadoop_do_classpath_subcommand CLASS "$@"
- ;;
- conftest)
- CLASS=org.apache.hadoop.util.ConfTest
- ;;
- credential)
- CLASS=org.apache.hadoop.security.alias.CredentialShell
- ;;
- daemonlog)
- CLASS=org.apache.hadoop.log.LogLevel
- ;;
- distch)
- CLASS=org.apache.hadoop.tools.DistCh
- hadoop_add_to_classpath_tools hadoop-extras
- ;;
- distcp)
- CLASS=org.apache.hadoop.tools.DistCp
- hadoop_add_to_classpath_tools hadoop-distcp
- ;;
- dtutil)
- CLASS=org.apache.hadoop.security.token.DtUtilShell
- ;;
- envvars)
- echo "JAVA_HOME='${JAVA_HOME}'"
- echo "HADOOP_COMMON_HOME='${HADOOP_COMMON_HOME}'"
- echo "HADOOP_COMMON_DIR='${HADOOP_COMMON_DIR}'"
- echo "HADOOP_COMMON_LIB_JARS_DIR='${HADOOP_COMMON_LIB_JARS_DIR}'"
- echo "HADOOP_COMMON_LIB_NATIVE_DIR='${HADOOP_COMMON_LIB_NATIVE_DIR}'"
- echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
- echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
- echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
- echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
- exit 0
- ;;
- fs)
- CLASS=org.apache.hadoop.fs.FsShell
- ;;
- gridmix)
- CLASS=org.apache.hadoop.mapred.gridmix.Gridmix
- hadoop_add_to_classpath_tools hadoop-rumen
- hadoop_add_to_classpath_tools hadoop-gridmix
- ;;
- jar)
- if [[ -n "${YARN_OPTS}" ]] || [[ -n "${YARN_CLIENT_OPTS}" ]]; then
- hadoop_error "WARNING: Use \"yarn jar\" to launch YARN applications."
- fi
- CLASS=org.apache.hadoop.util.RunJar
- ;;
- jnipath)
- hadoop_finalize
- echo "${JAVA_LIBRARY_PATH}"
- exit 0
- ;;
- kerbname)
- CLASS=org.apache.hadoop.security.HadoopKerberosName
- ;;
- key)
- CLASS=org.apache.hadoop.crypto.key.KeyShell
- ;;
- trace)
- CLASS=org.apache.hadoop.tracing.TraceAdmin
- ;;
- version)
- CLASS=org.apache.hadoop.util.VersionInfo
- ;;
- *)
- CLASS="${COMMAND}"
- if ! hadoop_validate_classname "${CLASS}"; then
- hadoop_exit_with_usage 1
- fi
- ;;
-esac
-
-hadoop_verify_user "${COMMAND}"
+HADOOP_SUBCMD_ARGS=("$@")
+
+if declare -f hadoop_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
+ hadoop_debug "Calling dynamically: hadoop_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}"
+ "hadoop_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+else
+ hadoopcmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+fi
+
+hadoop_verify_user "${HADOOP_SUBCMD}"
if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
- hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
+ hadoop_common_slave_mode_execute "${HADOOP_COMMON_HOME}/bin/hadoop" "${HADOOP_USER_PARAMS[@]}"
exit $?
fi
-# Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
-hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
+ HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
+ hadoop_verify_secure_prereq
+ hadoop_setup_secure_service
+ priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err"
+ priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+ daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+else
+ daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+fi
+
+if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
+ # shellcheck disable=SC2034
+ HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
+ if [[ -n "${HADOOP_SUBCMD_SECURESERVICE}" ]]; then
+ # shellcheck disable=SC2034
+ HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
+ else
+ # shellcheck disable=SC2034
+ HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
+ fi
+fi
hadoop_finalize
-hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
+if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then
+ if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
+ hadoop_secure_daemon_handler \
+ "${HADOOP_DAEMON_MODE}" \
+ "${HADOOP_SUBCMD}" \
+ "${HADOOP_CLASSNAME}" \
+ "${daemon_pidfile}" \
+ "${daemon_outfile}" \
+ "${priv_pidfile}" \
+ "${priv_outfile}" \
+ "${priv_errfile}" \
+ "${HADOOP_SUBCMD_ARGS[@]}"
+ else
+ hadoop_daemon_handler \
+ "${HADOOP_DAEMON_MODE}" \
+ "${HADOOP_SUBCMD}" \
+ "${HADOOP_CLASSNAME}" \
+ "${daemon_pidfile}" \
+ "${daemon_outfile}" \
+ "${HADOOP_SUBCMD_ARGS[@]}"
+ fi
+ exit $?
+else
+ # shellcheck disable=SC2086
+ hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}"
+fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
index a5fa10c..b6d7517 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
@@ -85,11 +85,11 @@ Apache Hadoop allows for third parties to easily add new features through a vari
Core to this functionality is the concept of a shell profile. Shell profiles are shell snippets that can do things such as add jars to the classpath, configure Java system properties and more.
-Shell profiles may be installed in either `${HADOOP_CONF_DIR}/shellprofile.d` or `${HADOOP_HOME}/libexec/shellprofile.d`. Shell profiles in the `libexec` directory are part of the base installation and cannot be overriden by the user. Shell profiles in the configuration directory may be ignored if the end user changes the configuration directory at runtime.
+Shell profiles may be installed in either `${HADOOP_CONF_DIR}/shellprofile.d` or `${HADOOP_HOME}/libexec/shellprofile.d`. Shell profiles in the `libexec` directory are part of the base installation and cannot be overridden by the user. Shell profiles in the configuration directory may be ignored if the end user changes the configuration directory at runtime.
An example of a shell profile is in the libexec directory.
-## Shell API
+### Shell API
Apache Hadoop's shell code has a [function library](./UnixShellAPI.html) that is open for administrators and developers to use to assist in their configuration and advanced feature management. These APIs follow the standard [Apache Hadoop Interface Classification](./InterfaceClassification.html), with one addition: Replaceable.
@@ -97,10 +97,8 @@ The shell code allows for core functions to be overridden. However, not all func
In order to replace a function, create a file called `hadoop-user-functions.sh` in the `${HADOOP_CONF_DIR}` directory. Simply define the new, replacement function in this file and the system will pick it up automatically. There may be as many replacement functions as needed in this file. Examples of function replacement are in the `hadoop-user-functions.sh.examples` file.
-
Functions that are marked Public and Stable are safe to use in shell profiles as-is. Other functions may change in a minor release.
-
### User-level API Access
In addition to `.hadoop-env`, which allows individual users to override `hadoop-env.sh`, user's may also use `.hadooprc`. This is called after the Apache Hadoop shell environment has been configured and allows the full set of shell API function calls.
@@ -112,3 +110,84 @@ hadoop_add_classpath /some/path/custom.jar
```
would go into `.hadooprc`
+
+### Dynamic Subcommands
+
+Utilizing the Shell API, it is possible for third parties to add their own subcommands to the primary Hadoop shell scripts (hadoop, hdfs, mapred, yarn).
+
+Prior to executing a subcommand, the primary scripts will check for the existence of a (scriptname)\_subcommand\_(subcommand) function. This function gets executed with the parameters set to all remaining command line arguments. For example, if the following function is defined:
+
+```bash
+function yarn_subcommand_hello
+{
+ echo "$@"
+ exit $?
+}
+```
+
+then executing `yarn --debug hello world I see you` will activate script debugging and call the `yarn_subcommand_hello` function as:
+
+```bash
+yarn_subcommand_hello world I see you
+```
+
+which will result in the output of:
+
+```bash
+world I see you
+```
+
+It is also possible to add the new subcommands to the usage output. The `hadoop_add_subcommand` function adds text to the usage output. Utilizing the standard HADOOP_SHELL_EXECNAME variable, we can limit which command gets our new function.
+
+```bash
+if [[ "${HADOOP_SHELL_EXECNAME}" = "yarn" ]]; then
+ hadoop_add_subcommand "hello" "Print some text to the screen"
+fi
+```
+
+This functionality may also be use to override the built-ins. For example, defining:
+
+```bash
+function hdfs_subcommand_fetchdt
+{
+ ...
+}
+```
+
+... will replace the existing `hdfs fetchdt` subcommand with a custom one.
+
+Some key environment variables related to Dynamic Subcommands:
+
+* HADOOP\_CLASSNAME
+
+This is the name of the Java class to use when program execution continues.
+
+* HADOOP\_SHELL\_EXECNAME
+
+This is the name of the script that is being executed. It will be one of hadoop, hdfs, mapred, or yarn.
+
+* HADOOP\_SUBCMD
+
+This is the subcommand that was passed on the command line.
+
+* HADOOP\_SUBCMD\_ARGS
+
+This array contains the argument list after the Apache Hadoop common argument processing has taken place and is the same list that is passed to the subcommand function as arguments. For example, if `hadoop --debug subcmd 1 2 3` has been executed on the command line, then `${HADOOP_SUBCMD_ARGS[0]}` will be 1 and `hadoop_subcommand_subcmd` will also have $1 equal to 1. This array list MAY be modified by subcommand functions to add or delete values from the argument list for further processing.
+
+* HADOOP\_SUBCMD\_SECURESERVICE
+
+If this command should/will be executed as a secure daemon, set this to true.
+
+* HADOOP\_SUBCMD\_SECUREUSER
+
+If this command should/will be executed as a secure daemon, set the user name to be used.
+
+* HADOOP\_SUBCMD\_SUPPORTDAEMONIZATION
+
+If this command can be executed as a daemon, set this to true.
+
+* HADOOP\_USER\_PARAMS
+
+This is the full content of the command line, prior to any parsing done. It will contain flags such as `--debug`. It MAY NOT be manipulated.
+
+The Apache Hadoop runtime facilities require functions exit if no further processing is required. For example, in the hello example above, Java and other facilities were not required so a simple `exit $?` was sufficient. However, if the function were to utilize `HADOOP_CLASSNAME`, then program execution must continue so that Java with the Apache Hadoop-specific parameters will be launched against the given Java class. Another example would be in the case of an unrecoverable error. It is the function's responsibility to print an appropriate message (preferably using the hadoop_error API call) and exit appropriately.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
index be2d7f5..cc37268 100755
--- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash
@@ -16,7 +16,7 @@
setup() {
- TMP=../../../target/test-dir/bats.$$.${RANDOM}
+ TMP="${BATS_TEST_DIRNAME}/../../../target/test-dir/bats.$$.${RANDOM}"
mkdir -p ${TMP}
TMP=$(cd -P -- "${TMP}" >/dev/null && pwd -P)
export TMP
@@ -38,7 +38,7 @@ setup() {
# shellcheck disable=SC2034
QATESTMODE=true
- . ../../main/bin/hadoop-functions.sh
+ . "${BATS_TEST_DIRNAME}/../../main/bin/hadoop-functions.sh"
pushd "${TMP}" >/dev/null
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommands.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommands.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommands.bats
new file mode 100755
index 0000000..c004a30
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommands.bats
@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+# the loading of shell profiles are tested elseswhere
+# this only tests the specific subcommand parts
+
+subcommandsetup () {
+ export HADOOP_LIBEXEC_DIR="${TMP}/libexec"
+ export HADOOP_CONF_DIR="${TMP}/conf"
+ mkdir -p "${HADOOP_LIBEXEC_DIR}"
+ echo ". \"${BATS_TEST_DIRNAME}/../../main/bin/hadoop-functions.sh\"" > "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+ cat <<-'TOKEN' >> "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+
+hadoop_subcommand_sub () {
+ echo "unittest"
+ exit 0
+}
+
+hadoop_subcommand_conftest ()
+{
+ echo conftest
+ exit 0
+}
+
+hadoop_subcommand_envcheck ()
+{
+ echo ${HADOOP_SHELL_EXECNAME}
+ exit 0
+}
+
+hadoop_subcommand_multi ()
+{
+ echo $2
+ exit 0
+}
+TOKEN
+ chmod a+rx "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+}
+
+@test "hadoop_subcommand (addition)" {
+ subcommandsetup
+ run "${BATS_TEST_DIRNAME}/../../main/bin/hadoop" sub
+ echo ">${output}<"
+ [ "${output}" = unittest ]
+}
+
+@test "hadoop_subcommand (substitute)" {
+ subcommandsetup
+ run "${BATS_TEST_DIRNAME}/../../main/bin/hadoop" conftest
+ echo ">${output}<"
+ [ "${output}" = conftest ]
+}
+
+@test "hadoop_subcommand (envcheck)" {
+ subcommandsetup
+ run "${BATS_TEST_DIRNAME}/../../main/bin/hadoop" envcheck
+ [ "${output}" = hadoop ]
+}
+
+@test "hadoop_subcommand (multiparams)" {
+ subcommandsetup
+ run "${BATS_TEST_DIRNAME}/../../main/bin/hadoop" multi 1 2
+ [ "${output}" = 2 ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 2d534bf..3696797 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -417,7 +417,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</plugins>
</build>
- <profiles>
+ <profiles>
<!-- profile that starts ApacheDS KDC server -->
<profile>
<id>startKdc</id>
@@ -559,5 +559,41 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</plugins>
</build>
</profile>
+
+ <!-- profile to test shell code -->
+ <profile>
+ <id>shelltest</id>
+ <activation>
+ <property>
+ <name>!skipTests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>hdfs-test-bats-driver</id>
+ <phase>test</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <exec dir="src/test/scripts"
+ executable="bash"
+ failonerror="true">
+ <arg value="./run-bats.sh" />
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index c365250..2a29d17 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -16,7 +16,12 @@
# limitations under the License.
MYNAME="${BASH_SOURCE-$0}"
+HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
+## @description build up the hdfs command's usage text.
+## @audience public
+## @stability stable
+## @replaceable no
function hadoop_usage
{
hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
@@ -56,7 +61,194 @@ function hadoop_usage
hadoop_add_subcommand "storagepolicies" "list/get/set block storage policies"
hadoop_add_subcommand "version" "print the version"
hadoop_add_subcommand "zkfc" "run the ZK Failover Controller daemon"
- hadoop_generate_usage "${MYNAME}" false
+ hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
+}
+
+## @description Default command handler for hadoop command
+## @audience public
+## @stability stable
+## @replaceable no
+## @param CLI arguments
+function hdfscmd_case
+{
+ subcmd=$1
+ shift
+
+ case ${subcmd} in
+ balancer)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.balancer.Balancer
+ hadoop_debug "Appending HADOOP_BALANCER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}"
+ ;;
+ cacheadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin
+ ;;
+ classpath)
+ hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
+ ;;
+ crypto)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CryptoAdmin
+ ;;
+ datanode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ # Determine if we're starting a secure datanode, and
+ # if so, redefine appropriate variables
+ if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
+ HADOOP_SUBCMD_SECURESERVICE="true"
+ HADOOP_SUBCMD_SECUREUSER="${HADOOP_SECURE_DN_USER}"
+
+ # backward compatiblity
+ HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}"
+ HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}"
+
+ hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
+ hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}"
+ HADOOP_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
+ else
+ hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
+ fi
+ ;;
+ debug)
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DebugAdmin'
+ ;;
+ dfs)
+ HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ dfsadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSAdmin
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ envvars)
+ echo "JAVA_HOME='${JAVA_HOME}'"
+ echo "HADOOP_HDFS_HOME='${HADOOP_HDFS_HOME}'"
+ echo "HDFS_DIR='${HDFS_DIR}'"
+ echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'"
+ echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
+ echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+ echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+ echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
+ exit 0
+ ;;
+ erasurecode)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ fetchdt)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
+ ;;
+ fsck)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ getconf)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetConf
+ ;;
+ groups)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetGroups
+ ;;
+ haadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSHAAdmin
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ journalnode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
+ hadoop_debug "Appending HADOOP_JOURNALNODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}"
+ ;;
+ jmxget)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
+ ;;
+ lsSnapshottableDir)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
+ ;;
+ mover)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.mover.Mover
+ hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}"
+ ;;
+ namenode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.NameNode'
+ hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}"
+ hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
+ ;;
+ nfs3)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then
+ HADOOP_SUBCMD_SECURESERVICE="true"
+ HADOOP_SUBCMD_SECUREUSER="${HADOOP_PRIVILEGED_NFS_USER}"
+
+ # backward compatiblity
+ HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}"
+ HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}"
+
+ hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
+ hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter
+ else
+ hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3
+ fi
+ ;;
+ oev)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
+ ;;
+ oiv)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
+ ;;
+ oiv_legacy)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
+ ;;
+ portmap)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap
+ hadoop_debug "Appending HADOOP_PORTMAP_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}"
+ ;;
+ secondarynamenode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
+ hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}"
+ hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
+ ;;
+ snapshotDiff)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
+ ;;
+ storagepolicies)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
+ ;;
+ version)
+ HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
+ ;;
+ zkfc)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
+ hadoop_debug "Appending HADOOP_ZKFC_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}"
+ ;;
+ *)
+ HADOOP_CLASSNAME="${subcmd}"
+ if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
+ hadoop_exit_with_usage 1
+ fi
+ ;;
+ esac
}
# let's locate libexec...
@@ -81,232 +273,76 @@ if [[ $# = 0 ]]; then
hadoop_exit_with_usage 1
fi
-COMMAND=$1
+HADOOP_SUBCMD=$1
shift
-case ${COMMAND} in
- balancer)
- supportdaemonization="true"
- CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
- hadoop_debug "Appending HADOOP_BALANCER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}"
- ;;
- cacheadmin)
- CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin
- ;;
- classpath)
- hadoop_do_classpath_subcommand CLASS "$@"
- ;;
- crypto)
- CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin
- ;;
- datanode)
- supportdaemonization="true"
- # Determine if we're starting a secure datanode, and
- # if so, redefine appropriate variables
- if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
- secure_service="true"
- secure_user="${HADOOP_SECURE_DN_USER}"
-
- # backward compatiblity
- HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}"
- HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}"
-
- hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
- hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}"
- CLASS="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
- else
- hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}"
- CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
- fi
- ;;
- debug)
- CLASS='org.apache.hadoop.hdfs.tools.DebugAdmin'
- ;;
- dfs)
- CLASS=org.apache.hadoop.fs.FsShell
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- dfsadmin)
- CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- envvars)
- echo "JAVA_HOME='${JAVA_HOME}'"
- echo "HADOOP_HDFS_HOME='${HADOOP_HDFS_HOME}'"
- echo "HDFS_DIR='${HDFS_DIR}'"
- echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'"
- echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
- echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
- echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
- echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
- exit 0
- ;;
- erasurecode)
- CLASS=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- fetchdt)
- CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
- ;;
- fsck)
- CLASS=org.apache.hadoop.hdfs.tools.DFSck
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- getconf)
- CLASS=org.apache.hadoop.hdfs.tools.GetConf
- ;;
- groups)
- CLASS=org.apache.hadoop.hdfs.tools.GetGroups
- ;;
- haadmin)
- CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- journalnode)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
- hadoop_debug "Appending HADOOP_JOURNALNODE_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}"
- ;;
- jmxget)
- CLASS=org.apache.hadoop.hdfs.tools.JMXGet
- ;;
- lsSnapshottableDir)
- CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
- ;;
- mover)
- supportdaemonization="true"
- CLASS=org.apache.hadoop.hdfs.server.mover.Mover
- hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}"
- ;;
- namenode)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
- hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}"
- hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
- ;;
- nfs3)
- supportdaemonization="true"
- if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then
- secure_service="true"
- secure_user="${HADOOP_PRIVILEGED_NFS_USER}"
+HADOOP_SUBCMD_ARGS=("$@")
- # backward compatiblity
- HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}"
- HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}"
-
- hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
- hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}"
- CLASS=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter
- else
- hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}"
- CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3
- fi
- ;;
- oev)
- CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
- ;;
- oiv)
- CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
- ;;
- oiv_legacy)
- CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
- ;;
- portmap)
- supportdaemonization="true"
- CLASS=org.apache.hadoop.portmap.Portmap
- hadoop_debug "Appending HADOOP_PORTMAP_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}"
- ;;
- secondarynamenode)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
- hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}"
- hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
- ;;
- snapshotDiff)
- CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
- ;;
- storagepolicies)
- CLASS=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
- ;;
- version)
- CLASS=org.apache.hadoop.util.VersionInfo
- ;;
- zkfc)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
- hadoop_debug "Appending HADOOP_ZKFC_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}"
- ;;
- *)
- CLASS="${COMMAND}"
- if ! hadoop_validate_classname "${CLASS}"; then
- hadoop_exit_with_usage 1
- fi
- ;;
-esac
+if declare -f hdfs_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
+ hadoop_debug "Calling dynamically: hdfs_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}"
+ "hdfs_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+else
+ hdfscmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+fi
-hadoop_verify_user "${COMMAND}"
+hadoop_verify_user "${HADOOP_SUBCMD}"
if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
exit $?
fi
-if [[ -n "${secure_service}" ]]; then
- HADOOP_SECURE_USER="${secure_user}"
+if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
+ HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
hadoop_verify_secure_prereq
hadoop_setup_secure_service
- priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
- priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.err"
- priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
- daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
- daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
+ priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err"
+ priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+ daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
else
- daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
- daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
+ daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
fi
if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
# shellcheck disable=SC2034
HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
- if [[ -n "${secure_service}" ]]; then
+ if [[ -n "${HADOOP_SUBCMD_SECURESERVICE}" ]]; then
# shellcheck disable=SC2034
- HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
+ HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
else
# shellcheck disable=SC2034
- HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
+ HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
fi
fi
hadoop_finalize
-if [[ -n "${supportdaemonization}" ]]; then
- if [[ -n "${secure_service}" ]]; then
+if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then
+ if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
hadoop_secure_daemon_handler \
- "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\
- "${daemon_pidfile}" "${daemon_outfile}" \
- "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@"
+ "${HADOOP_DAEMON_MODE}" \
+ "${HADOOP_SUBCMD}" \
+ "${HADOOP_CLASSNAME}" \
+ "${daemon_pidfile}" \
+ "${daemon_outfile}" \
+ "${priv_pidfile}" \
+ "${priv_outfile}" \
+ "${priv_errfile}" \
+ "${HADOOP_SUBCMD_ARGS[@]}"
else
- hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\
- "${daemon_pidfile}" "${daemon_outfile}" "$@"
+ hadoop_daemon_handler \
+ "${HADOOP_DAEMON_MODE}" \
+ "${HADOOP_SUBCMD}" \
+ "${HADOOP_CLASSNAME}" \
+ "${daemon_pidfile}" \
+ "${daemon_outfile}" \
+ "${HADOOP_SUBCMD_ARGS[@]}"
fi
exit $?
else
# shellcheck disable=SC2086
- hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
+ hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hadoop-hdfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hadoop-hdfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hadoop-hdfs.sh
new file mode 100644
index 0000000..398f8a0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hadoop-hdfs.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop_add_profile hdfs
+
+function _hdfs_hadoop_classpath
+{
+ #
+ # get all of the hdfs jars+config in the path
+ #
+ # developers
+ if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
+ hadoop_add_classpath "${HADOOP_HDFS_HOME}/hadoop-hdfs/target/classes"
+ fi
+
+ # put hdfs in classpath if present
+ if [[ -d "${HADOOP_HDFS_HOME}/${HDFS_DIR}/webapps" ]]; then
+ hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"
+ fi
+
+ hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR}"'/*'
+ hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"'/*'
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hdfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hdfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hdfs.sh
deleted file mode 100644
index 5eb9e48..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hdfs.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-hadoop_add_profile hdfs
-
-function _hdfs_hadoop_classpath
-{
- #
- # get all of the hdfs jars+config in the path
- #
- # developers
- if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/hadoop-hdfs/target/classes"
- fi
-
- # put hdfs in classpath if present
- if [[ -d "${HADOOP_HDFS_HOME}/${HDFS_DIR}/webapps" ]]; then
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"
- fi
-
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR}"'/*'
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"'/*'
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/hdfs-functions_test_helper.bash
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/hdfs-functions_test_helper.bash b/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/hdfs-functions_test_helper.bash
new file mode 100755
index 0000000..d3cdda4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/hdfs-functions_test_helper.bash
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+setup() {
+
+ TMP="${BATS_TEST_DIRNAME}/../../../target/test-dir/bats.$$.${RANDOM}"
+ mkdir -p "${TMP}"
+ TMP=$(cd -P -- "${TMP}" >/dev/null && pwd -P)
+ export TMP
+ TESTBINDIR="${BATS_TEST_DIRNAME}"
+ HADOOP_LIBEXEC_DIR=${TESTBINDIR}/../../main/bin
+ HADOOP_LIBEXEC_DIR=$(cd -P -- "${HADOOP_LIBEXEC_DIR}" >/dev/null && pwd -P)
+
+ # shellcheck disable=SC2034
+ HADOOP_SHELL_SCRIPT_DEBUG=true
+ unset HADOOP_CONF_DIR
+ # we unset both of these for bw compat
+ unset HADOOP_HOME
+ unset HADOOP_PREFIX
+
+ echo "bindir: ${TESTBINDIR}" 2>&1
+
+ mkdir -p "${TMP}"
+
+ # shellcheck disable=SC2034
+ QATESTMODE=true
+
+ # shellcheck disable=SC1090
+ . "${BATS_TEST_DIRNAME}/../../../../../hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh"
+ pushd "${TMP}" >/dev/null
+}
+
+teardown() {
+ popd >/dev/null
+ rm -rf "${TMP}"
+}
+
+
+strstr() {
+ if [ "${1#*$2}" != "${1}" ]; then
+ echo true
+ else
+ echo false
+ fi
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/hdfs_subcommands.bats
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/hdfs_subcommands.bats b/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/hdfs_subcommands.bats
new file mode 100755
index 0000000..d074ab9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/hdfs_subcommands.bats
@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hdfs-functions_test_helper
+
+# the loading of shell profiles are tested elseswhere
+# this only tests the specific subcommand parts
+
+subcommandsetup () {
+ export HADOOP_LIBEXEC_DIR="${TMP}/libexec"
+ export HADOOP_CONF_DIR="${TMP}/conf"
+ mkdir -p "${HADOOP_LIBEXEC_DIR}"
+ echo ". \"${BATS_TEST_DIRNAME}/../../../../../hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh\"" > "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
+ cat <<-'TOKEN' >> "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
+
+hdfs_subcommand_sub () {
+ echo "unittest"
+ exit 0
+}
+
+hdfs_subcommand_cacheadmin ()
+{
+ echo cacheadmin
+ exit 0
+}
+
+hdfs_subcommand_envcheck ()
+{
+ echo ${HADOOP_SHELL_EXECNAME}
+ exit 0
+}
+
+hdfs_subcommand_multi ()
+{
+ echo $2
+ exit 0
+}
+TOKEN
+ chmod a+rx "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
+}
+
+@test "hdfs_subcommand (addition)" {
+ subcommandsetup
+ run "${BATS_TEST_DIRNAME}/../../main/bin/hdfs" sub
+ echo ">${output}<"
+ [ "${output}" = unittest ]
+}
+
+@test "hdfs_subcommand (substitute)" {
+ subcommandsetup
+ run "${BATS_TEST_DIRNAME}/../../main/bin/hdfs" cacheadmin
+ echo ">${output}<"
+ [ "${output}" = cacheadmin ]
+}
+
+@test "hdfs_subcommand (envcheck)" {
+ subcommandsetup
+ run "${BATS_TEST_DIRNAME}/../../main/bin/hdfs" envcheck
+ [ "${output}" = hdfs ]
+}
+
+@test "hdfs_subcommand (multiparams)" {
+ subcommandsetup
+ run "${BATS_TEST_DIRNAME}/../../main/bin/hdfs" multi 1 2
+ [ "${output}" = 2 ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/run-bats.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/run-bats.sh b/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/run-bats.sh
new file mode 100755
index 0000000..566f47a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/scripts/run-bats.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+targetdir=../../../target
+mkdir -p ${targetdir}/surefire-reports ${targetdir}/tap
+
+batsexe=$(which bats) 2>/dev/null
+
+if [[ -z ${batsexe} ]]; then
+ echo "not ok - no bats executable found" > "${targetdir}/tap/shelltest.tap"
+ echo ""
+ echo ""
+ echo "ERROR: bats not installed. Skipping bash tests."
+ echo "ERROR: Please install bats as soon as possible."
+ echo ""
+ echo ""
+ exit 0
+fi
+
+for j in *.bats; do
+ echo Running bats -t "${j}"
+ bats -t "${j}" 2>&1 | tee "${targetdir}/tap/${j}.tap"
+ result=${PIPESTATUS[0]}
+ ((exitcode=exitcode+result))
+done
+
+if [[ ${exitcode} -gt 0 ]]; then
+ exit 1
+fi
+exit 0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index f280f31..046d48c 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -16,13 +16,15 @@
# limitations under the License.
MYNAME="${BASH_SOURCE-$0}"
+HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
+## @description build up the mapred command's usage text.
+## @audience public
+## @stability stable
+## @replaceable no
function hadoop_usage
{
- hadoop_add_subcommand "archive" "create a hadoop archive"
- hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives"
hadoop_add_subcommand "classpath" "prints the class path needed for running mapreduce subcommands"
- hadoop_add_subcommand "distcp" "copy file or directories recursively"
hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
hadoop_add_subcommand "historyserver" "run job history servers as a standalone daemon"
hadoop_add_subcommand "hsadmin" "job history server admin interface"
@@ -31,7 +33,85 @@ function hadoop_usage
hadoop_add_subcommand "queue" "get information regarding JobQueues"
hadoop_add_subcommand "sampler" "sampler"
hadoop_add_subcommand "version" "print the version"
- hadoop_generate_usage "${MYNAME}" true
+ hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
+}
+
+## @description Default command handler for hadoop command
+## @audience public
+## @stability stable
+## @replaceable no
+## @param CLI arguments
+function mapredcmd_case
+{
+ subcmd=$1
+ shift
+
+ case ${subcmd} in
+ mradmin|jobtracker|tasktracker|groups)
+ hadoop_error "Sorry, the ${subcmd} command is no longer supported."
+ hadoop_error "You may find similar functionality with the \"yarn\" shell command."
+ hadoop_exit_with_usage 1
+ ;;
+ classpath)
+ hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
+ ;;
+ envvars)
+ echo "JAVA_HOME='${JAVA_HOME}'"
+ echo "HADOOP_MAPRED_HOME='${HADOOP_MAPRED_HOME}'"
+ echo "MAPRED_DIR='${MAPRED_DIR}'"
+ echo "MAPRED_LIB_JARS_DIR='${MAPRED_LIB_JARS_DIR}'"
+ echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
+ echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+ echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+ echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
+ exit 0
+ ;;
+ historyserver)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
+ hadoop_debug "Appending HADOOP_JOB_HISTORYSERVER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOB_HISTORYSERVER_OPTS}"
+ if [ -n "${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}" ]; then
+ # shellcheck disable=SC2034
+ HADOOP_HEAPSIZE_MAX="${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}"
+ fi
+ HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_JHS_LOGGER:-$HADOOP_DAEMON_ROOT_LOGGER}
+ ;;
+ hsadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ job)
+ HADOOP_CLASSNAME=org.apache.hadoop.mapred.JobClient
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ pipes)
+ HADOOP_CLASSNAME=org.apache.hadoop.mapred.pipes.Submitter
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ queue)
+ HADOOP_CLASSNAME=org.apache.hadoop.mapred.JobQueueClient
+ ;;
+ sampler)
+ HADOOP_CLASSNAME=org.apache.hadoop.mapred.lib.InputSampler
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ version)
+ HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ *)
+ HADOOP_CLASSNAME="${subcmd}"
+ if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
+ hadoop_exit_with_usage 1
+ fi
+ ;;
+ esac
}
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
@@ -58,125 +138,71 @@ if [ $# = 0 ]; then
hadoop_exit_with_usage 1
fi
-COMMAND=$1
+HADOOP_SUBCMD=$1
shift
-case ${COMMAND} in
- mradmin|jobtracker|tasktracker|groups)
- hadoop_error "Sorry, the ${COMMAND} command is no longer supported."
- hadoop_error "You may find similar functionality with the \"yarn\" shell command."
- hadoop_exit_with_usage 1
- ;;
- archive)
- CLASS=org.apache.hadoop.tools.HadoopArchives
- hadoop_add_to_classpath_tools hadoop-archives
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- archive-logs)
- CLASS=org.apache.hadoop.tools.HadoopArchiveLogs
- hadoop_add_to_classpath_tools hadoop-archive-logs
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- classpath)
- hadoop_do_classpath_subcommand CLASS "$@"
- ;;
- distcp)
- CLASS=org.apache.hadoop.tools.DistCp
- hadoop_add_to_classpath_tools hadoop-distcp
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- envvars)
- echo "JAVA_HOME='${JAVA_HOME}'"
- echo "HADOOP_MAPRED_HOME='${HADOOP_MAPRED_HOME}'"
- echo "MAPRED_DIR='${MAPRED_DIR}'"
- echo "MAPRED_LIB_JARS_DIR='${MAPRED_LIB_JARS_DIR}'"
- echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
- echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
- echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
- echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
- exit 0
- ;;
- historyserver)
- supportdaemonization="true"
- CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
- hadoop_debug "Appending HADOOP_JOB_HISTORYSERVER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOB_HISTORYSERVER_OPTS}"
- if [ -n "${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}" ]; then
- HADOOP_HEAPSIZE_MAX="${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}"
- fi
- HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_JHS_LOGGER:-$HADOOP_DAEMON_ROOT_LOGGER}
- ;;
- hsadmin)
- CLASS=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- job)
- CLASS=org.apache.hadoop.mapred.JobClient
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- pipes)
- CLASS=org.apache.hadoop.mapred.pipes.Submitter
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- queue)
- CLASS=org.apache.hadoop.mapred.JobQueueClient
- ;;
- sampler)
- CLASS=org.apache.hadoop.mapred.lib.InputSampler
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- version)
- CLASS=org.apache.hadoop.util.VersionInfo
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
- ;;
- *)
- CLASS="${COMMAND}"
- if ! hadoop_validate_classname "${CLASS}"; then
- hadoop_exit_with_usage 1
- fi
- ;;
-esac
+HADOOP_SUBCMD_ARGS=("$@")
-hadoop_verify_user "${COMMAND}"
+if declare -f mapred_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
+ hadoop_debug "Calling dynamically: mapred_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}"
+ "mapred_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+else
+ mapredcmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
+fi
+
+hadoop_verify_user "${HADOOP_SUBCMD}"
if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
hadoop_common_slave_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}"
exit $?
fi
-daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
-daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
-
+if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
+ HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
+ hadoop_verify_secure_prereq
+ hadoop_setup_secure_service
+ priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err"
+ priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+ daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+else
+ daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
+ daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
+fi
if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
# shellcheck disable=SC2034
HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
hadoop_add_param HADOOP_OPTS mapred.jobsummary.logger "-Dmapred.jobsummary.logger=${HADOOP_ROOT_LOGGER}"
# shellcheck disable=SC2034
- HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
+ HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
fi
hadoop_finalize
-if [[ -n "${supportdaemonization}" ]]; then
- if [[ -n "${secure_service}" ]]; then
- hadoop_secure_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}"\
- "${CLASS}" "${daemon_pidfile}" "${daemon_outfile}" \
- "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@"
+if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then
+ if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
+ hadoop_secure_daemon_handler \
+ "${HADOOP_DAEMON_MODE}" \
+ "${HADOOP_SUBCMD}" \
+ "${HADOOP_CLASSNAME}" \
+ "${daemon_pidfile}" \
+ "${daemon_outfile}" \
+ "${priv_pidfile}" \
+ "${priv_outfile}" \
+ "${priv_errfile}" \
+ "${HADOOP_SUBCMD_ARGS[@]}"
else
- hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}" \
- "${daemon_pidfile}" "${daemon_outfile}" "$@"
+ hadoop_daemon_handler \
+ "${HADOOP_DAEMON_MODE}" \
+ "${HADOOP_SUBCMD}" \
+ "${HADOOP_CLASSNAME}" \
+ "${daemon_pidfile}" \
+ "${daemon_outfile}" \
+ "${HADOOP_SUBCMD_ARGS[@]}"
fi
exit $?
else
- hadoop_java_exec "${COMMAND}" "${CLASS}" "$@"
+ hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-mapreduce-project/shellprofile.d/hadoop-mapreduce.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/shellprofile.d/hadoop-mapreduce.sh b/hadoop-mapreduce-project/shellprofile.d/hadoop-mapreduce.sh
new file mode 100644
index 0000000..b0e3adb
--- /dev/null
+++ b/hadoop-mapreduce-project/shellprofile.d/hadoop-mapreduce.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop_add_profile mapred
+
+function _mapred_hadoop_classpath
+{
+ #
+ # get all of the mapreduce jars+config in the path
+ #
+ # developers
+ if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-shuffle/target/classes"
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-common/target/classes"
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs/target/classes"
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs-plugins/target/classes"
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-app/target/classes"
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-jobclient/target/classes"
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-core/target/classes"
+ fi
+
+ if [[ -d "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}/webapps" ]]; then
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"
+ fi
+
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_LIB_JARS_DIR}"'/*'
+ hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"'/*'
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-mapreduce-project/shellprofile.d/mapreduce.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/shellprofile.d/mapreduce.sh b/hadoop-mapreduce-project/shellprofile.d/mapreduce.sh
deleted file mode 100644
index 0b3dab1..0000000
--- a/hadoop-mapreduce-project/shellprofile.d/mapreduce.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-hadoop_add_profile mapred
-
-function _mapred_hadoop_classpath
-{
- #
- # get all of the mapreduce jars+config in the path
- #
- # developers
- if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-shuffle/target/classes"
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-common/target/classes"
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs/target/classes"
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs-plugins/target/classes"
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-app/target/classes"
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-jobclient/target/classes"
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-core/target/classes"
- fi
-
- if [[ -d "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}/webapps" ]]; then
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"
- fi
-
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_LIB_JARS_DIR}"'/*'
- hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"'/*'
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
new file mode 100755
index 0000000..ae7b6c6
--- /dev/null
+++ b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f mapred_subcommand_archive-logs >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+ hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives"
+ fi
+
+ # this can't be indented otherwise shelldocs won't get it
+
+## @description archive-logs command for mapred
+## @audience public
+## @stability stable
+## @replaceable yes
+function mapred_subcommand_archive-logs
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs
+ hadoop_add_to_classpath_tools hadoop-archive-logs
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh b/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh
new file mode 100755
index 0000000..f74fe5b
--- /dev/null
+++ b/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f hadoop_subcommand_archive >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+ hadoop_add_subcommand "archive" "create a Hadoop archive"
+ fi
+
+ # this can't be indented otherwise shelldocs won't get it
+
+## @description archive command for hadoop (and mapred)
+## @audience public
+## @stability stable
+## @replaceable yes
+function hadoop_subcommand_archive
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives
+ hadoop_add_to_classpath_tools hadoop-archives
+}
+
+fi
+
+if ! declare -f mapred_subcommand_archive >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+ hadoop_add_subcommand "archive" "create a Hadoop archive"
+ fi
+
+ # this can't be indented otherwise shelldocs won't get it
+
+## @description archive command for mapred (calls hadoop version)
+## @audience public
+## @stability stable
+## @replaceable yes
+function mapred_subcommand_archive
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives
+ hadoop_add_to_classpath_tools hadoop-archives
+}
+
+fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
new file mode 100755
index 0000000..0178c54
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f hadoop_subcommand_distcp >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+ hadoop_add_subcommand "distcp" "copy file or directories recursively"
+ fi
+
+ # this can't be indented otherwise shelldocs won't get it
+
+## @description distcp command for hadoop
+## @audience public
+## @stability stable
+## @replaceable yes
+function hadoop_subcommand_distcp
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
+ hadoop_add_to_classpath_tools hadoop-distcp
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+fi
+
+if ! declare -f mapred_subcommand_distcp >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then
+ hadoop_add_subcommand "distcp" "copy file or directories recursively"
+ fi
+
+ # this can't be indented otherwise shelldocs won't get it
+
+## @description distcp command for mapred (calls hadoop)
+## @audience public
+## @stability stable
+## @replaceable yes
+function mapred_subcommand_distcp
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
+ hadoop_add_to_classpath_tools hadoop-distcp
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/730bc746/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
new file mode 100755
index 0000000..829d406
--- /dev/null
+++ b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if ! declare -f hadoop_subcommand_distch >/dev/null 2>/dev/null; then
+
+ if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then
+ hadoop_add_subcommand "distch" "distributed metadata changer"
+ fi
+
+ # this can't be indented otherwise shelldocs won't get it
+
+## @description distch command for hadoop
+## @audience public
+## @stability stable
+## @replaceable yes
+function hadoop_subcommand_distch
+{
+ # shellcheck disable=SC2034
+ HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCh
+ hadoop_add_to_classpath_tools hadoop-extras
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+}
+
+fi
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HADOOP-13149. Windows distro build
fails on dist-copynativelibs. Contributed by Chris Nauroth.
Posted by ar...@apache.org.
HADOOP-13149. Windows distro build fails on dist-copynativelibs. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/576e2d12
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/576e2d12
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/576e2d12
Branch: refs/heads/HDFS-7240
Commit: 576e2d1278a584ef8e804832c852c3f8323bb64a
Parents: b9685e8
Author: Chris Nauroth <cn...@apache.org>
Authored: Mon May 16 12:05:02 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Mon May 16 12:05:02 2016 -0700
----------------------------------------------------------------------
hadoop-project-dist/pom.xml | 33 +++++++++++++++++----------------
1 file changed, 17 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/576e2d12/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index a554b42..2cccbbe 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -325,22 +325,23 @@
<goal>exec</goal>
</goals>
<configuration>
- <executable>${project.parent.basedir}/../dev-support/bin/dist-copynativelibs</executable>
- <workingDirectory>${project.build.directory}</workingDirectory>
- <requiresOnline>false</requiresOnline>
- <arguments>
- <argument>--version=${project.version}</argument>
- <argument>--builddir=${project.build.directory}</argument>
- <argument>--artifactid=${project.artifactId}</argument>
- <argument>--isalbundle=${bundle.isal}</argument>
- <argument>--isallib=${isal.lib}</argument>
- <argument>--openssllib=${openssl.lib}</argument>
- <argument>--opensslbinbundle=${bundle.openssl.in.bin}</argument>
- <argument>--openssllibbundle=${bundle.openssl}</argument>
- <argument>--snappybinbundle=${bundle.snappy.in.bin}</argument>
- <argument>--snappylib=${snappy.lib}</argument>
- <argument>--snappylibbundle=${bundle.snappy}</argument>
- </arguments>
+ <executable>${shell-executable}</executable>
+ <workingDirectory>${project.build.directory}</workingDirectory>
+ <requiresOnline>false</requiresOnline>
+ <arguments>
+ <argument>${project.parent.basedir}/../dev-support/bin/dist-copynativelibs</argument>
+ <argument>--version=${project.version}</argument>
+ <argument>--builddir=${project.build.directory}</argument>
+ <argument>--artifactid=${project.artifactId}</argument>
+ <argument>--isalbundle=${bundle.isal}</argument>
+ <argument>--isallib=${isal.lib}</argument>
+ <argument>--openssllib=${openssl.lib}</argument>
+ <argument>--opensslbinbundle=${bundle.openssl.in.bin}</argument>
+ <argument>--openssllibbundle=${bundle.openssl}</argument>
+ <argument>--snappybinbundle=${bundle.snappy.in.bin}</argument>
+ <argument>--snappylib=${snappy.lib}</argument>
+ <argument>--snappylibbundle=${bundle.snappy}</argument>
+ </arguments>
</configuration>
</execution>
</executions>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-5100. The
YarnApplicationState is always running in ATS even application is finished.
Contributed by Xuan Gong.
Posted by ar...@apache.org.
YARN-5100. The YarnApplicationState is always running in ATS even application is finished. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/141873ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/141873ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/141873ca
Branch: refs/heads/HDFS-7240
Commit: 141873ca7d68619e3458712b17aed57fec8fd7c8
Parents: 2120de5
Author: Junping Du <ju...@apache.org>
Authored: Thu May 19 09:13:29 2016 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Thu May 19 09:13:29 2016 -0700
----------------------------------------------------------------------
.../ApplicationHistoryManagerOnTimelineStore.java | 15 +++++++++++++--
...TestApplicationHistoryManagerOnTimelineStore.java | 12 ++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/141873ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 51e2356..aedf6f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -358,6 +358,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
createdTime = event.getTimestamp();
} else if (event.getEventType().equals(
ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) {
+ // TODO: YARN-5101. This type of events are parsed in
+ // time-stamp descending order which means the previous event
+ // could override the information from the later same type of event.
Map<String, Object> eventInfo = event.getEventInfo();
if (eventInfo == null) {
continue;
@@ -376,8 +379,10 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
}
if (eventInfo.containsKey(
ApplicationMetricsConstants.STATE_EVENT_INFO)) {
- state = YarnApplicationState.valueOf(eventInfo.get(
- ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
+ if (!isFinalState(state)) {
+ state = YarnApplicationState.valueOf(eventInfo.get(
+ ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
+ }
}
} else if (event.getEventType().equals(
ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
@@ -429,6 +434,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
amNodeLabelExpression), appViewACLs);
}
+ private static boolean isFinalState(YarnApplicationState state) {
+ return state == YarnApplicationState.FINISHED
+ || state == YarnApplicationState.FAILED
+ || state == YarnApplicationState.KILLED;
+ }
+
private static ApplicationAttemptReport convertToApplicationAttemptReport(
TimelineEntity entity) {
String host = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/141873ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 3c97584..b65b22b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -537,6 +537,18 @@ public class TestApplicationHistoryManagerOnTimelineStore {
}
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
+ // send a YARN_APPLICATION_STATE_UPDATED event
+ // after YARN_APPLICATION_FINISHED
+ // The final YarnApplicationState should not be changed
+ tEvent = new TimelineEvent();
+ tEvent.setEventType(
+ ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 4L + appId.getId());
+ eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
+ YarnApplicationState.KILLED);
+ tEvent.setEventInfo(eventInfo);
+ entity.addEvent(tEvent);
if (enableUpdateEvent) {
tEvent = new TimelineEvent();
createAppModifiedEvent(appId, tEvent, "changed queue", 5);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: Addendum patch for YARN-4832.
Contributed by Junping Du
Posted by ar...@apache.org.
Addendum patch for YARN-4832. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c6726e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c6726e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c6726e2
Branch: refs/heads/HDFS-7240
Commit: 0c6726e20d9503589b21123f30757ddfbd405dde
Parents: f6ef876
Author: Jian He <ji...@apache.org>
Authored: Tue May 17 14:49:20 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Tue May 17 14:50:30 2016 -0700
----------------------------------------------------------------------
.../src/main/proto/yarn_server_common_service_protos.proto | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c6726e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 7d39d30..a7e5a86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -101,8 +101,8 @@ message NodeHeartbeatResponseProto {
optional bool areNodeLabelsAcceptedByRM = 11 [default = false];
repeated ContainerProto containers_to_decrease = 12;
repeated SignalContainerRequestProto containers_to_signal = 13;
- optional ContainerQueuingLimitProto container_queuing_limit = 14;
- optional ResourceProto resource = 15;
+ optional ResourceProto resource = 14;
+ optional ContainerQueuingLimitProto container_queuing_limit = 15;
}
message ContainerQueuingLimitProto {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-13148. TestDistCpViewFs to
include IOExceptions in test error reports. Contributed by Steve Loughran.
Posted by ar...@apache.org.
HADOOP-13148. TestDistCpViewFs to include IOExceptions in test error reports. Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9685e85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9685e85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9685e85
Branch: refs/heads/HDFS-7240
Commit: b9685e85d59e69e5dd64802fa9025dad1b315be5
Parents: 81effb7
Author: Chris Nauroth <cn...@apache.org>
Authored: Mon May 16 11:53:17 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Mon May 16 11:53:17 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/tools/TestDistCpViewFs.java | 93 +++++---------------
1 file changed, 22 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9685e85/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
index a6939a2..5511e09 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
@@ -73,13 +73,12 @@ public class TestDistCpViewFs {
}
@Test
- public void testSingleFileMissingTarget() {
+ public void testSingleFileMissingTarget() throws IOException {
caseSingleFileMissingTarget(false);
caseSingleFileMissingTarget(true);
}
-
- private void caseSingleFileMissingTarget(boolean sync) {
+ private void caseSingleFileMissingTarget(boolean sync) throws IOException{
try {
addEntries(listFile, "singlefile1/file1");
@@ -88,21 +87,18 @@ public class TestDistCpViewFs {
runTest(listFile, target, false, sync);
checkResult(target, 1);
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testSingleFileTargetFile() {
+ public void testSingleFileTargetFile() throws IOException{
caseSingleFileTargetFile(false);
caseSingleFileTargetFile(true);
}
- private void caseSingleFileTargetFile(boolean sync) {
+ private void caseSingleFileTargetFile(boolean sync) throws IOException {
try {
addEntries(listFile, "singlefile1/file1");
@@ -111,21 +107,18 @@ public class TestDistCpViewFs {
runTest(listFile, target, false, sync);
checkResult(target, 1);
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testSingleFileTargetDir() {
+ public void testSingleFileTargetDir() throws IOException {
caseSingleFileTargetDir(false);
caseSingleFileTargetDir(true);
}
- private void caseSingleFileTargetDir(boolean sync) {
+ private void caseSingleFileTargetDir(boolean sync) throws IOException {
try {
addEntries(listFile, "singlefile2/file2");
@@ -135,21 +128,18 @@ public class TestDistCpViewFs {
runTest(listFile, target, true, sync);
checkResult(target, 1, "file2");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testSingleDirTargetMissing() {
+ public void testSingleDirTargetMissing() throws IOException{
caseSingleDirTargetMissing(false);
caseSingleDirTargetMissing(true);
}
- private void caseSingleDirTargetMissing(boolean sync) {
+ private void caseSingleDirTargetMissing(boolean sync) throws IOException {
try {
addEntries(listFile, "singledir");
@@ -158,16 +148,13 @@ public class TestDistCpViewFs {
runTest(listFile, target, false, sync);
checkResult(target, 1, "dir1");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testSingleDirTargetPresent() {
+ public void testSingleDirTargetPresent() throws IOException{
try {
addEntries(listFile, "singledir");
@@ -177,16 +164,13 @@ public class TestDistCpViewFs {
runTest(listFile, target, true, false);
checkResult(target, 1, "singledir/dir1");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testUpdateSingleDirTargetPresent() {
+ public void testUpdateSingleDirTargetPresent() throws IOException {
try {
addEntries(listFile, "Usingledir");
@@ -196,21 +180,18 @@ public class TestDistCpViewFs {
runTest(listFile, target, true, true);
checkResult(target, 1, "Udir1");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testMultiFileTargetPresent() {
+ public void testMultiFileTargetPresent() throws IOException {
caseMultiFileTargetPresent(false);
caseMultiFileTargetPresent(true);
}
- private void caseMultiFileTargetPresent(boolean sync) {
+ private void caseMultiFileTargetPresent(boolean sync) throws IOException {
try {
addEntries(listFile, "multifile/file3", "multifile/file4", "multifile/file5");
@@ -220,21 +201,18 @@ public class TestDistCpViewFs {
runTest(listFile, target, true, sync);
checkResult(target, 3, "file3", "file4", "file5");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testMultiFileTargetMissing() {
+ public void testMultiFileTargetMissing() throws IOException {
caseMultiFileTargetMissing(false);
caseMultiFileTargetMissing(true);
}
- private void caseMultiFileTargetMissing(boolean sync) {
+ private void caseMultiFileTargetMissing(boolean sync) throws IOException {
try {
addEntries(listFile, "multifile/file3", "multifile/file4", "multifile/file5");
@@ -243,16 +221,13 @@ public class TestDistCpViewFs {
runTest(listFile, target, false, sync);
checkResult(target, 3, "file3", "file4", "file5");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testMultiDirTargetPresent() {
+ public void testMultiDirTargetPresent() throws IOException {
try {
addEntries(listFile, "multifile", "singledir");
@@ -262,16 +237,13 @@ public class TestDistCpViewFs {
runTest(listFile, target, true, false);
checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5", "singledir/dir1");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testUpdateMultiDirTargetPresent() {
+ public void testUpdateMultiDirTargetPresent() throws IOException {
try {
addEntries(listFile, "Umultifile", "Usingledir");
@@ -281,16 +253,13 @@ public class TestDistCpViewFs {
runTest(listFile, target, true, true);
checkResult(target, 4, "Ufile3", "Ufile4", "Ufile5", "Udir1");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testMultiDirTargetMissing() {
+ public void testMultiDirTargetMissing() throws IOException {
try {
addEntries(listFile, "multifile", "singledir");
@@ -301,16 +270,13 @@ public class TestDistCpViewFs {
checkResult(target, 2, "multifile/file3", "multifile/file4",
"multifile/file5", "singledir/dir1");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testUpdateMultiDirTargetMissing() {
+ public void testUpdateMultiDirTargetMissing() throws IOException {
try {
addEntries(listFile, "multifile", "singledir");
@@ -320,16 +286,13 @@ public class TestDistCpViewFs {
runTest(listFile, target, false, true);
checkResult(target, 4, "file3", "file4", "file5", "dir1");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
}
}
@Test
- public void testGlobTargetMissingSingleLevel() {
+ public void testGlobTargetMissingSingleLevel() throws IOException {
try {
Path listFile = new Path("target/tmp1/listing").makeQualified(fs.getUri(),
@@ -342,9 +305,6 @@ public class TestDistCpViewFs {
checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5",
"singledir/dir2/file6");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
TestDistCpUtils.delete(fs, "target/tmp1");
@@ -352,7 +312,7 @@ public class TestDistCpViewFs {
}
@Test
- public void testUpdateGlobTargetMissingSingleLevel() {
+ public void testUpdateGlobTargetMissingSingleLevel() throws IOException {
try {
Path listFile = new Path("target/tmp1/listing").makeQualified(fs.getUri(),
@@ -364,9 +324,6 @@ public class TestDistCpViewFs {
runTest(listFile, target, false, true);
checkResult(target, 4, "file3", "file4", "file5", "dir2/file6");
- } catch (IOException e) {
- LOG.error("Exception encountered while running distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
TestDistCpUtils.delete(fs, "target/tmp1");
@@ -374,7 +331,7 @@ public class TestDistCpViewFs {
}
@Test
- public void testGlobTargetMissingMultiLevel() {
+ public void testGlobTargetMissingMultiLevel() throws IOException {
try {
Path listFile = new Path("target/tmp1/listing").makeQualified(fs.getUri(),
@@ -388,9 +345,6 @@ public class TestDistCpViewFs {
checkResult(target, 4, "file3", "file4", "file5",
"dir3/file7", "dir3/file8", "dir3/file9");
- } catch (IOException e) {
- LOG.error("Exception encountered while running distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
TestDistCpUtils.delete(fs, "target/tmp1");
@@ -398,7 +352,7 @@ public class TestDistCpViewFs {
}
@Test
- public void testUpdateGlobTargetMissingMultiLevel() {
+ public void testUpdateGlobTargetMissingMultiLevel() throws IOException {
try {
Path listFile = new Path("target/tmp1/listing").makeQualified(fs.getUri(),
@@ -412,9 +366,6 @@ public class TestDistCpViewFs {
checkResult(target, 6, "file3", "file4", "file5",
"file7", "file8", "file9");
- } catch (IOException e) {
- LOG.error("Exception encountered while running distcp", e);
- Assert.fail("distcp failure");
} finally {
TestDistCpUtils.delete(fs, root);
TestDistCpUtils.delete(fs, "target/tmp1");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-13140.
FileSystem#initialize must not attempt to create StorageStatistics objects
with null or empty schemes (Mingliang Liu via cmccabe)
Posted by ar...@apache.org.
HADOOP-13140. FileSystem#initialize must not attempt to create StorageStatistics objects with null or empty schemes (Mingliang Liu via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2c1d918
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2c1d918
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2c1d918
Branch: refs/heads/HDFS-7240
Commit: f2c1d9181f80e53890b50309b15c9ea37cb24987
Parents: 8a9ecb7
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed May 18 09:15:32 2016 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed May 18 09:15:32 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/FileSystem.java | 11 ++++++++++-
.../org/apache/hadoop/fs/GlobalStorageStatistics.java | 5 ++++-
2 files changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2c1d918/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 574027e..9e13a7a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -75,6 +75,7 @@ import org.apache.htrace.core.TraceScope;
import com.google.common.annotations.VisibleForTesting;
+import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
/****************************************************************
@@ -211,7 +212,13 @@ public abstract class FileSystem extends Configured implements Closeable {
* @param conf the configuration
*/
public void initialize(URI name, Configuration conf) throws IOException {
- statistics = getStatistics(name.getScheme(), getClass());
+ final String scheme;
+ if (name.getScheme() == null || name.getScheme().isEmpty()) {
+ scheme = getDefaultUri(conf).getScheme();
+ } else {
+ scheme = name.getScheme();
+ }
+ statistics = getStatistics(scheme, getClass());
resolveSymlinks = conf.getBoolean(
CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
@@ -3590,6 +3597,8 @@ public abstract class FileSystem extends Configured implements Closeable {
@Deprecated
public static synchronized Statistics getStatistics(final String scheme,
Class<? extends FileSystem> cls) {
+ checkArgument(scheme != null,
+ "No statistics is allowed for a file system with null scheme!");
Statistics result = statisticsTable.get(cls);
if (result == null) {
final Statistics newStats = new Statistics(scheme);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2c1d918/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java
index f22e78c..7502965 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java
@@ -23,6 +23,7 @@ import java.util.NavigableMap;
import java.util.NoSuchElementException;
import java.util.TreeMap;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
/**
@@ -55,7 +56,7 @@ public enum GlobalStorageStatistics {
* null if there is none.
*/
public synchronized StorageStatistics get(String name) {
- return map.get(name);
+ return name == null ? null : map.get(name);
}
/**
@@ -70,6 +71,8 @@ public enum GlobalStorageStatistics {
*/
public synchronized StorageStatistics put(String name,
StorageStatisticsProvider provider) {
+ Preconditions.checkNotNull(name,
+ "Storage statistics can not have a null name!");
StorageStatistics stats = map.get(name);
if (stats != null) {
return stats;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HADOOP-13157. Follow-on
improvements to hadoop credential commands. Contributed by Mike Yoder.
Posted by ar...@apache.org.
HADOOP-13157. Follow-on improvements to hadoop credential commands. Contributed by Mike Yoder.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7154ace7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7154ace7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7154ace7
Branch: refs/heads/HDFS-7240
Commit: 7154ace71212e9fb9dd6209a92165fb075df7806
Parents: dd99f5f
Author: Andrew Wang <wa...@apache.org>
Authored: Tue May 17 17:44:30 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue May 17 17:44:30 2016 -0700
----------------------------------------------------------------------
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 79 ++++----------------
.../org/apache/hadoop/crypto/key/KeyShell.java | 9 ++-
.../apache/hadoop/security/ProviderUtils.java | 75 +++++++++++++++++++
.../alias/AbstractJavaKeyStoreProvider.java | 77 ++++---------------
.../hadoop/security/alias/CredentialShell.java | 6 +-
.../apache/hadoop/crypto/key/TestKeyShell.java | 11 +--
.../hadoop/security/alias/TestCredShell.java | 11 +--
7 files changed, 127 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154ace7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index c7c1779..1827c27 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.crypto.key;
import com.google.common.base.Preconditions;
-import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -38,12 +37,10 @@ import com.google.common.annotations.VisibleForTesting;
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
-import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.net.URI;
-import java.net.URL;
import java.security.GeneralSecurityException;
import java.security.Key;
import java.security.KeyStore;
@@ -139,43 +136,14 @@ public class JavaKeyStoreProvider extends KeyProvider {
}
/**
- * The password is either found in the environment or in a file. This
- * routine implements the logic for locating the password in these
- * locations.
- * @return The password as a char []; null if not found.
- * @throws IOException
- */
- private char[] locatePassword() throws IOException {
- char[] pass = null;
- // Get the password file from the conf, if not present from the user's
- // environment var
- if (System.getenv().containsKey(KEYSTORE_PASSWORD_ENV_VAR)) {
- pass = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray();
- }
- if (pass == null) {
- String pwFile = getConf().get(KEYSTORE_PASSWORD_FILE_KEY);
- if (pwFile != null) {
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
- URL pwdFile = cl.getResource(pwFile);
- if (pwdFile == null) {
- // Provided Password file does not exist
- throw new IOException("Password file does not exists");
- }
- try (InputStream is = pwdFile.openStream()) {
- pass = IOUtils.toString(is).trim().toCharArray();
- }
- }
- }
- return pass;
- }
-
- /**
* Open up and initialize the keyStore.
- * @throws IOException
+ * @throws IOException If there is a problem reading the password file
+ * or a problem reading the keystore.
*/
private void locateKeystore() throws IOException {
try {
- password = locatePassword();
+ password = ProviderUtils.locatePassword(KEYSTORE_PASSWORD_ENV_VAR,
+ getConf().get(KEYSTORE_PASSWORD_FILE_KEY));
if (password == null) {
password = KEYSTORE_PASSWORD_DEFAULT;
}
@@ -331,48 +299,31 @@ public class JavaKeyStoreProvider extends KeyProvider {
}
}
- private Path constructNewPath(Path path) {
- Path newPath = new Path(path.toString() + "_NEW");
- return newPath;
+ private static Path constructNewPath(Path path) {
+ return new Path(path.toString() + "_NEW");
}
- private Path constructOldPath(Path path) {
- Path oldPath = new Path(path.toString() + "_OLD");
- return oldPath;
+ private static Path constructOldPath(Path path) {
+ return new Path(path.toString() + "_OLD");
}
@Override
public boolean needsPassword() throws IOException {
- return (null == locatePassword());
- }
+ return (null == ProviderUtils.locatePassword(KEYSTORE_PASSWORD_ENV_VAR,
+ getConf().get(KEYSTORE_PASSWORD_FILE_KEY)));
- @VisibleForTesting
- public static final String NO_PASSWORD_WARN =
- "WARNING: You have accepted the use of the default provider password\n" +
- "by not configuring a password in one of the two following locations:\n";
- public static final String NO_PASSWORD_ERROR =
- "ERROR: The provider cannot find a password in the expected " +
- "locations.\nPlease supply a password using one of the " +
- "following two mechanisms:\n";
- @VisibleForTesting public static final String NO_PASSWORD_INSTRUCTIONS =
- " o In the environment variable " +
- KEYSTORE_PASSWORD_ENV_VAR + "\n" +
- " o In a file referred to by the configuration entry\n" +
- " " + KEYSTORE_PASSWORD_FILE_KEY + ".\n" +
- "Please review the documentation regarding provider passwords at\n" +
- "http://hadoop.apache.org/docs/current/hadoop-project-dist/" +
- "hadoop-common/CredentialProviderAPI.html#Keystore_Passwords\n";
- @VisibleForTesting public static final String NO_PASSWORD_CONT =
- "Continuing with the default provider password.\n";
+ }
@Override
public String noPasswordWarning() {
- return NO_PASSWORD_WARN + NO_PASSWORD_INSTRUCTIONS + NO_PASSWORD_CONT;
+ return ProviderUtils.noPasswordWarning(KEYSTORE_PASSWORD_ENV_VAR,
+ KEYSTORE_PASSWORD_FILE_KEY);
}
@Override
public String noPasswordError() {
- return NO_PASSWORD_ERROR + NO_PASSWORD_INSTRUCTIONS;
+ return ProviderUtils.noPasswordError(KEYSTORE_PASSWORD_ENV_VAR,
+ KEYSTORE_PASSWORD_FILE_KEY);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154ace7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
index fcc2105..5de8aa8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
@@ -47,7 +47,8 @@ public class KeyShell extends Configured implements Tool {
" [" + DeleteCommand.USAGE + "]\n" +
" [" + ListCommand.USAGE + "]\n";
private static final String LIST_METADATA = "keyShell.list.metadata";
- @VisibleForTesting public static final String NO_VALID_PROVIDERS =
+ @VisibleForTesting
+ public static final String NO_VALID_PROVIDERS =
"There are no valid (non-transient) providers configured.\n" +
"No action has been taken. Use the -provider option to specify\n" +
"a provider. If you want to use a transient provider then you\n" +
@@ -60,9 +61,11 @@ public class KeyShell extends Configured implements Tool {
private boolean strict = false;
/** allows stdout to be captured if necessary. */
- @VisibleForTesting public PrintStream out = System.out;
+ @VisibleForTesting
+ public PrintStream out = System.out;
/** allows stderr to be captured if necessary. */
- @VisibleForTesting public PrintStream err = System.err;
+ @VisibleForTesting
+ public PrintStream err = System.err;
private boolean userSuppliedProvider = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154ace7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
index ae08fba..84ee912 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
@@ -19,9 +19,13 @@
package org.apache.hadoop.security;
import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URL;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -36,6 +40,23 @@ import org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider;
*
*/
public final class ProviderUtils {
+ @VisibleForTesting
+ public static final String NO_PASSWORD_WARN =
+ "WARNING: You have accepted the use of the default provider password\n" +
+ "by not configuring a password in one of the two following locations:\n";
+ @VisibleForTesting
+ public static final String NO_PASSWORD_ERROR =
+ "ERROR: The provider cannot find a password in the expected " +
+ "locations.\nPlease supply a password using one of the " +
+ "following two mechanisms:\n";
+ @VisibleForTesting
+ public static final String NO_PASSWORD_CONT =
+ "Continuing with the default provider password.\n";
+ @VisibleForTesting
+ public static final String NO_PASSWORD_INSTRUCTIONS_DOC =
+ "Please review the documentation regarding provider passwords in\n" +
+ "the keystore passwords section of the Credential Provider API\n";
+
private static final Log LOG = LogFactory.getLog(ProviderUtils.class);
/**
@@ -174,4 +195,58 @@ public final class ProviderUtils {
}
return conf;
}
+
+ /**
+ * The password is either found in the environment or in a file. This
+ * routine implements the logic for locating the password in these
+ * locations.
+ *
+ * @param envWithPass The name of the environment variable that might
+ * contain the password. Must not be null.
+ * @param fileWithPass The name of a file that could contain the password.
+ * Can be null.
+ * @return The password as a char []; null if not found.
+ * @throws IOException If fileWithPass is non-null and points to a
+ * nonexistent file or a file that fails to open and be read properly.
+ */
+ public static char[] locatePassword(String envWithPass, String fileWithPass)
+ throws IOException {
+ char[] pass = null;
+ // Get the password file from the conf, if not present from the user's
+ // environment var
+ if (System.getenv().containsKey(envWithPass)) {
+ pass = System.getenv(envWithPass).toCharArray();
+ }
+ if (pass == null) {
+ if (fileWithPass != null) {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ URL pwdFile = cl.getResource(fileWithPass);
+ if (pwdFile == null) {
+ // Provided Password file does not exist
+ throw new IOException("Password file does not exist");
+ }
+ try (InputStream is = pwdFile.openStream()) {
+ pass = IOUtils.toString(is).trim().toCharArray();
+ }
+ }
+ }
+ return pass;
+ }
+
+ private static String noPasswordInstruction(String envKey, String fileKey) {
+ return
+ " * In the environment variable " + envKey + "\n" +
+ " * In a file referred to by the configuration entry\n" +
+ " " + fileKey + ".\n" +
+ NO_PASSWORD_INSTRUCTIONS_DOC;
+ }
+
+ public static String noPasswordWarning(String envKey, String fileKey) {
+ return NO_PASSWORD_WARN + noPasswordInstruction(envKey, fileKey) +
+ NO_PASSWORD_CONT;
+ }
+
+ public static String noPasswordError(String envKey, String fileKey) {
+ return NO_PASSWORD_ERROR + noPasswordInstruction(envKey, fileKey);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154ace7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
index 895b715..335c198 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
@@ -18,10 +18,8 @@
package org.apache.hadoop.security.alias;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -34,7 +32,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
-import java.net.URL;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.KeyStoreException;
@@ -64,11 +61,11 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
public static final Log LOG = LogFactory.getLog(
AbstractJavaKeyStoreProvider.class);
- public static final String CREDENTIAL_PASSWORD_NAME =
+ public static final String CREDENTIAL_PASSWORD_ENV_VAR =
"HADOOP_CREDSTORE_PASSWORD";
- public static final String KEYSTORE_PASSWORD_FILE_KEY =
+ public static final String CREDENTIAL_PASSWORD_FILE_KEY =
"hadoop.security.credstore.java-keystore-provider.password-file";
- public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
+ public static final String CREDENTIAL_PASSWORD_DEFAULT = "none";
private Path path;
private final URI uri;
@@ -303,44 +300,17 @@ public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
}
/**
- * The password is either found in the environment or in a file. This
- * routine implements the logic for locating the password in these
- * locations.
- *
- * @return The password as a char []; null if not found.
- * @throws IOException
- */
- private char[] locatePassword() throws IOException {
- char[] pass = null;
- if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
- pass = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
- }
- // if not in ENV get check for file
- if (pass == null) {
- String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
- if (pwFile != null) {
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
- URL pwdFile = cl.getResource(pwFile);
- if (pwdFile != null) {
- try (InputStream is = pwdFile.openStream()) {
- pass = IOUtils.toString(is).trim().toCharArray();
- }
- }
- }
- }
- return pass;
- }
-
- /**
* Open up and initialize the keyStore.
*
- * @throws IOException
+ * @throws IOException If there is a problem reading the password file
+ * or a problem reading the keystore.
*/
private void locateKeystore() throws IOException {
try {
- password = locatePassword();
+ password = ProviderUtils.locatePassword(CREDENTIAL_PASSWORD_ENV_VAR,
+ conf.get(CREDENTIAL_PASSWORD_FILE_KEY));
if (password == null) {
- password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
+ password = CREDENTIAL_PASSWORD_DEFAULT.toCharArray();
}
KeyStore ks;
ks = KeyStore.getInstance("jceks");
@@ -364,38 +334,21 @@ public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
@Override
public boolean needsPassword() throws IOException {
- return (null == locatePassword());
- }
+ return (null == ProviderUtils.locatePassword(CREDENTIAL_PASSWORD_ENV_VAR,
+ conf.get(CREDENTIAL_PASSWORD_FILE_KEY)));
- @VisibleForTesting
- public static final String NO_PASSWORD_WARN =
- "WARNING: You have accepted the use of the default provider password\n" +
- "by not configuring a password in one of the two following locations:\n";
- @VisibleForTesting
- public static final String NO_PASSWORD_ERROR =
- "ERROR: The provider cannot find a password in the expected " +
- "locations.\nPlease supply a password using one of the " +
- "following two mechanisms:\n";
- @VisibleForTesting
- public static final String NO_PASSWORD_INSTRUCTIONS =
- " o In the environment variable " +
- CREDENTIAL_PASSWORD_NAME + "\n" +
- " o In a file referred to by the configuration entry\n" +
- " " + KEYSTORE_PASSWORD_FILE_KEY + ".\n" +
- "Please review the documentation regarding provider passwords at\n" +
- "http://hadoop.apache.org/docs/current/hadoop-project-dist/" +
- "hadoop-common/CredentialProviderAPI.html#Keystore_Passwords\n";
- @VisibleForTesting public static final String NO_PASSWORD_CONT =
- "Continuing with the default provider password.\n";
+ }
@Override
public String noPasswordWarning() {
- return NO_PASSWORD_WARN + NO_PASSWORD_INSTRUCTIONS + NO_PASSWORD_CONT;
+ return ProviderUtils.noPasswordWarning(CREDENTIAL_PASSWORD_ENV_VAR,
+ CREDENTIAL_PASSWORD_FILE_KEY);
}
@Override
public String noPasswordError() {
- return NO_PASSWORD_ERROR + NO_PASSWORD_INSTRUCTIONS;
+ return ProviderUtils.noPasswordError(CREDENTIAL_PASSWORD_ENV_VAR,
+ CREDENTIAL_PASSWORD_FILE_KEY);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154ace7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
index a047ac1..c15de90 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
@@ -58,9 +58,11 @@ public class CredentialShell extends Configured implements Tool {
private boolean strict = false;
/** Allows stdout to be captured if necessary. */
- @VisibleForTesting public PrintStream out = System.out;
+ @VisibleForTesting
+ public PrintStream out = System.out;
/** Allows stderr to be captured if necessary. */
- @VisibleForTesting public PrintStream err = System.err;
+ @VisibleForTesting
+ public PrintStream err = System.err;
private boolean userSuppliedProvider = false;
private String value = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154ace7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
index eaf064e..8966643 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
@@ -25,6 +25,7 @@ import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Before;
@@ -116,11 +117,11 @@ public class TestKeyShell {
assertTrue(outContent.toString().contains(keyName + " has been " +
"successfully created"));
assertTrue(outContent.toString()
- .contains(JavaKeyStoreProvider.NO_PASSWORD_WARN));
+ .contains(ProviderUtils.NO_PASSWORD_WARN));
assertTrue(outContent.toString()
- .contains(JavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
+ .contains(ProviderUtils.NO_PASSWORD_INSTRUCTIONS_DOC));
assertTrue(outContent.toString()
- .contains(JavaKeyStoreProvider.NO_PASSWORD_CONT));
+ .contains(ProviderUtils.NO_PASSWORD_CONT));
String listOut = listKeys(ks, false);
assertTrue(listOut.contains(keyName));
@@ -240,9 +241,9 @@ public class TestKeyShell {
rc = ks.run(args1);
assertEquals(1, rc);
assertTrue(outContent.toString()
- .contains(JavaKeyStoreProvider.NO_PASSWORD_ERROR));
+ .contains(ProviderUtils.NO_PASSWORD_ERROR));
assertTrue(outContent.toString()
- .contains(JavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
+ .contains(ProviderUtils.NO_PASSWORD_INSTRUCTIONS_DOC));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7154ace7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
index 5ba7cf0..569fe73 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
@@ -30,6 +30,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Test;
@@ -64,11 +65,11 @@ public class TestCredShell {
assertTrue(outContent.toString().contains("credential1 has been successfully " +
"created."));
assertTrue(outContent.toString()
- .contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_WARN));
+ .contains(ProviderUtils.NO_PASSWORD_WARN));
assertTrue(outContent.toString()
- .contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
+ .contains(ProviderUtils.NO_PASSWORD_INSTRUCTIONS_DOC));
assertTrue(outContent.toString()
- .contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_CONT));
+ .contains(ProviderUtils.NO_PASSWORD_CONT));
outContent.reset();
String[] args2 = {"list", "-provider",
@@ -246,9 +247,9 @@ public class TestCredShell {
assertFalse(outContent.toString().contains("credential1 has been " +
"successfully created."));
assertTrue(outContent.toString()
- .contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_ERROR));
+ .contains(ProviderUtils.NO_PASSWORD_ERROR));
assertTrue(outContent.toString()
- .contains(AbstractJavaKeyStoreProvider.NO_PASSWORD_INSTRUCTIONS));
+ .contains(ProviderUtils.NO_PASSWORD_INSTRUCTIONS_DOC));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: MAPREDUCE-6701. application master
log can not be available when clicking jobhistory's am logs link. Contributed
by Haibo Chen
Posted by ar...@apache.org.
MAPREDUCE-6701. application master log can not be available when clicking jobhistory's am logs link. Contributed by Haibo Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12fa4ec1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12fa4ec1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12fa4ec1
Branch: refs/heads/HDFS-7240
Commit: 12fa4ec141f8d0cb105b6f60c3739a4b26890fd5
Parents: 4a5819d
Author: Jason Lowe <jl...@apache.org>
Authored: Tue May 17 14:54:13 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue May 17 14:54:13 2016 +0000
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java | 2 +-
.../mapreduce/v2/hs/webapp/dao/AMAttemptInfo.java | 2 +-
.../mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java | 12 +++++++-----
3 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fa4ec1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
index ddac853..13774a8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java
@@ -142,7 +142,7 @@ public class HsJobBlock extends HtmlBlock {
td().a(".nodelink", url(MRWebAppUtil.getYARNWebappScheme(),
attempt.getNodeHttpAddress()),
attempt.getNodeHttpAddress())._().
- td().a(".logslink", url(attempt.getShortLogsLink()),
+ td().a(".logslink", url(attempt.getLogsLink()),
"logs")._().
_();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fa4ec1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/AMAttemptInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/AMAttemptInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/AMAttemptInfo.java
index 065f8c7..54383df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/AMAttemptInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/AMAttemptInfo.java
@@ -68,7 +68,7 @@ public class AMAttemptInfo {
if (containerId != null) {
this.containerId = containerId.toString();
this.logsLink = join(host, pathPrefix,
- ujoin("logs", this.nodeHttpAddress, this.containerId, jobId, user));
+ ujoin("logs", this.nodeId, this.containerId, jobId, user));
this.shortLogsLink = ujoin("logs", this.nodeHttpAddress, this.containerId,
jobId, user);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12fa4ec1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
index fc67c1f..4c4088f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
@@ -766,6 +766,7 @@ public class TestHsWebServicesJobs extends JerseyTest {
for (int i = 0; i < attempts.length(); i++) {
JSONObject attempt = attempts.getJSONObject(i);
verifyHsJobAttemptsGeneric(job, attempt.getString("nodeHttpAddress"),
+ attempt.getString("nodeId"),
attempt.getInt("id"),
attempt.getLong("startTime"), attempt.getString("containerId"),
attempt.getString("logsLink"));
@@ -779,6 +780,7 @@ public class TestHsWebServicesJobs extends JerseyTest {
Element element = (Element) nodes.item(i);
verifyHsJobAttemptsGeneric(job,
WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
+ WebServicesTestUtils.getXmlString(element, "nodeId"),
WebServicesTestUtils.getXmlInt(element, "id"),
WebServicesTestUtils.getXmlLong(element, "startTime"),
WebServicesTestUtils.getXmlString(element, "containerId"),
@@ -787,7 +789,8 @@ public class TestHsWebServicesJobs extends JerseyTest {
}
public void verifyHsJobAttemptsGeneric(Job job, String nodeHttpAddress,
- int id, long startTime, String containerId, String logsLink) {
+ String nodeId, int id, long startTime, String containerId,
+ String logsLink) {
boolean attemptFound = false;
for (AMInfo amInfo : job.getAMInfos()) {
if (amInfo.getAppAttemptId().getAttemptId() == id) {
@@ -801,10 +804,9 @@ public class TestHsWebServicesJobs extends JerseyTest {
WebServicesTestUtils.checkStringMatch("containerId", amInfo
.getContainerId().toString(), containerId);
- String localLogsLink = join(
- "hsmockwebapp",
- ujoin("logs", nodeHttpAddress, containerId,
- MRApps.toString(job.getID()), job.getUserName()));
+ String localLogsLink = join("hsmockwebapp",
+ ujoin("logs", nodeId, containerId, MRApps.toString(job.getID()),
+ job.getUserName()));
assertTrue("logsLink", logsLink.contains(localLogsLink));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDFS-10410.
RedundantEditLogInputStream.LOG is set to wrong class. (John Zhuge via lei)
Posted by ar...@apache.org.
HDFS-10410. RedundantEditLogInputStream.LOG is set to wrong class. (John Zhuge via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a6e74ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a6e74ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a6e74ac
Branch: refs/heads/HDFS-7240
Commit: 6a6e74acf5c38a4995c4622148721cfe2f1fbdad
Parents: 4b55642
Author: Lei Xu <le...@apache.org>
Authored: Mon May 16 17:05:46 2016 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Mon May 16 17:05:46 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a6e74ac/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
index 33be8b0..a73206b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RedundantEditLogInputStream.java
@@ -37,7 +37,8 @@ import com.google.common.primitives.Longs;
* different subset of the available edits.
*/
class RedundantEditLogInputStream extends EditLogInputStream {
- public static final Log LOG = LogFactory.getLog(EditLogInputStream.class.getName());
+ public static final Log LOG = LogFactory.getLog(
+ RedundantEditLogInputStream.class.getName());
private int curIdx;
private long prevTxId;
private final EditLogInputStream[] streams;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDFS-10242. Cannot create space
quota of zero. Contributed by Takashi Ohnishi.
Posted by ar...@apache.org.
HDFS-10242. Cannot create space quota of zero. Contributed by Takashi Ohnishi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9fe5828f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9fe5828f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9fe5828f
Branch: refs/heads/HDFS-7240
Commit: 9fe5828f05accc6746cb08a43916f7557dac533a
Parents: ccc93e7
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue May 17 15:53:09 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue May 17 15:53:09 2016 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/QuotaUsage.java | 4 ++--
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 2 +-
.../src/test/java/org/apache/hadoop/hdfs/TestQuota.java | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe5828f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
index 9ffab2f..9ad6a28 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
@@ -296,7 +296,7 @@ public class QuotaUsage {
quotaStr = formatSize(quota, hOption);
quotaRem = formatSize(quota-fileAndDirectoryCount, hOption);
}
- if (spaceQuota > 0) {
+ if (spaceQuota >= 0) {
spaceQuotaStr = formatSize(spaceQuota, hOption);
spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
}
@@ -314,7 +314,7 @@ public class QuotaUsage {
String quotaStr = QUOTA_NONE;
String quotaRem = QUOTA_INF;
- if (typeQuota > 0) {
+ if (typeQuota >= 0) {
quotaStr = formatSize(typeQuota, hOption);
quotaRem = formatSize(typeQuota - typeConsumed, hOption);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe5828f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index d4e3187..aa48f7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2368,7 +2368,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
if ((namespaceQuota <= 0 &&
namespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
namespaceQuota != HdfsConstants.QUOTA_RESET) ||
- (storagespaceQuota <= 0 &&
+ (storagespaceQuota < 0 &&
storagespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
storagespaceQuota != HdfsConstants.QUOTA_RESET)) {
throw new IllegalArgumentException("Invalid values for quota : " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe5828f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index 2c55156..c3bed00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -63,7 +63,7 @@ public class TestQuota {
throws Exception {
int val = admin.run(args);
if (expectEror) {
- assertEquals(val, -1);
+ assertEquals(-1, val);
} else {
assertTrue(val>=0);
}
@@ -263,7 +263,7 @@ public class TestQuota {
// 16a: set the quota of /test to be 0
args = new String[]{"-setQuota", "0", parent.toString()};
runCommand(admin, args, true);
- runCommand(admin, true, "-setSpaceQuota", "0", args[2]);
+ runCommand(admin, false, "-setSpaceQuota", "0", args[2]);
// 16b: set the quota of /test to be -1
args[1] = "-1";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Posted by ar...@apache.org.
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76279f58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76279f58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76279f58
Branch: refs/heads/HDFS-7240
Commit: 76279f5877d592d2cc0816654f30cf1bfcf3aae1
Parents: 4770ae7 141873c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu May 19 12:44:16 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu May 19 12:44:16 2016 -0700
----------------------------------------------------------------------
BUILDING.txt | 8 +-
dev-support/docker/Dockerfile | 26 +-
hadoop-assemblies/pom.xml | 4 +-
.../main/resources/assemblies/hadoop-tools.xml | 56 ++
hadoop-build-tools/pom.xml | 2 +-
hadoop-client/pom.xml | 4 +-
.../hadoop-annotations/pom.xml | 4 +-
.../hadoop-auth-examples/pom.xml | 4 +-
hadoop-common-project/hadoop-auth/pom.xml | 4 +-
.../authentication/util/KerberosName.java | 9 +-
.../TestKerberosAuthenticationHandler.java | 7 +-
.../authentication/util/TestKerberosName.java | 17 +-
hadoop-common-project/hadoop-common/pom.xml | 4 +-
.../hadoop-common/src/main/bin/hadoop | 317 +++++---
.../src/main/bin/hadoop-functions.sh | 23 +
.../org/apache/hadoop/conf/Configuration.java | 21 +
.../hadoop/crypto/key/JavaKeyStoreProvider.java | 89 ++-
.../apache/hadoop/crypto/key/KeyProvider.java | 32 +
.../org/apache/hadoop/crypto/key/KeyShell.java | 127 +--
.../fs/CommonConfigurationKeysPublic.java | 7 +-
.../hadoop/fs/EmptyStorageStatistics.java | 43 +
.../org/apache/hadoop/fs/FSDataInputStream.java | 9 +
.../java/org/apache/hadoop/fs/FileSystem.java | 58 +-
.../hadoop/fs/FileSystemStorageStatistics.java | 136 ++++
.../main/java/org/apache/hadoop/fs/FsShell.java | 15 +-
.../hadoop/fs/GlobalStorageStatistics.java | 130 +++
.../java/org/apache/hadoop/fs/QuotaUsage.java | 4 +-
.../org/apache/hadoop/fs/StorageStatistics.java | 93 +++
.../apache/hadoop/fs/TrashPolicyDefault.java | 1 -
.../hadoop/fs/UnionStorageStatistics.java | 113 +++
.../org/apache/hadoop/io/DataInputBuffer.java | 81 +-
.../main/java/org/apache/hadoop/io/IOUtils.java | 4 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 7 +-
.../hadoop/io/compress/DecompressorStream.java | 41 +-
.../hadoop/io/retry/FailoverProxyProvider.java | 11 +
.../hadoop/io/retry/RetryInvocationHandler.java | 350 +++++----
.../apache/hadoop/ipc/DecayRpcScheduler.java | 15 +-
.../java/org/apache/hadoop/log/LogLevel.java | 16 +-
.../hadoop/metrics2/MetricStringBuilder.java | 141 ++++
.../hadoop/metrics2/lib/MutableCounterLong.java | 2 +-
.../org/apache/hadoop/net/NetworkTopology.java | 144 +++-
.../java/org/apache/hadoop/net/NodeBase.java | 9 +
.../java/org/apache/hadoop/security/KDiag.java | 46 +-
.../hadoop/security/LdapGroupsMapping.java | 269 +++++--
.../apache/hadoop/security/ProviderUtils.java | 75 ++
.../hadoop/security/ShellBasedIdMapping.java | 21 +-
.../hadoop/security/UserGroupInformation.java | 6 +-
.../alias/AbstractJavaKeyStoreProvider.java | 113 +--
.../security/alias/CredentialProvider.java | 37 +-
.../hadoop/security/alias/CredentialShell.java | 166 ++--
.../security/alias/JavaKeyStoreProvider.java | 6 +-
.../alias/LocalJavaKeyStoreProvider.java | 4 +-
.../security/http/RestCsrfPreventionFilter.java | 12 +-
.../security/http/XFrameOptionsFilter.java | 167 ++++
.../hadoop/security/http/package-info.java | 22 +
.../org/apache/hadoop/security/token/Token.java | 2 +-
.../AbstractDelegationTokenIdentifier.java | 33 +-
.../DelegationTokenAuthenticationFilter.java | 7 +-
.../token/delegation/web/ServletUtils.java | 7 +-
.../org/apache/hadoop/util/SysInfoLinux.java | 39 +-
.../src/main/resources/core-default.xml | 68 +-
.../src/site/markdown/CommandsManual.md | 22 +-
.../src/site/markdown/FileSystemShell.md | 5 +
.../src/site/markdown/GroupsMapping.md | 6 +
.../src/site/markdown/SecureMode.md | 6 +
.../src/site/markdown/UnixShellGuide.md | 87 +-
.../crypto/key/TestKeyProviderFactory.java | 2 +-
.../apache/hadoop/crypto/key/TestKeyShell.java | 33 +-
.../apache/hadoop/fs/TestFilterFileSystem.java | 1 +
.../org/apache/hadoop/fs/TestHarFileSystem.java | 1 +
.../fs/contract/AbstractBondedFSContract.java | 3 +-
.../hadoop/io/TestSequenceFileAppend.java | 67 ++
.../hadoop/io/compress/FakeCompressor.java | 113 +++
.../hadoop/io/compress/FakeDecompressor.java | 109 +++
.../compress/TestBlockDecompressorStream.java | 180 +----
.../io/compress/TestDecompressorStream.java | 99 +++
.../apache/hadoop/io/retry/TestRetryProxy.java | 67 +-
.../org/apache/hadoop/log/TestLogLevel.java | 39 +-
.../org/apache/hadoop/security/TestKDiag.java | 16 +
.../hadoop/security/TestLdapGroupsMapping.java | 33 +-
.../security/TestLdapGroupsMappingBase.java | 71 +-
.../TestLdapGroupsMappingWithOneQuery.java | 100 +++
.../TestLdapGroupsMappingWithPosixGroup.java | 41 +-
.../security/TestUserGroupInformation.java | 27 +-
.../hadoop/security/alias/TestCredShell.java | 56 +-
.../alias/TestCredentialProviderFactory.java | 5 +-
.../http/TestRestCsrfPreventionFilter.java | 40 +-
.../security/http/TestXFrameOptionsFilter.java | 151 ++++
.../java/org/apache/hadoop/util/JarFinder.java | 26 +-
.../java/org/apache/hadoop/util/TestRunJar.java | 29 +-
.../apache/hadoop/util/TestSysInfoLinux.java | 92 ++-
.../scripts/hadoop-functions_test_helper.bash | 4 +-
.../src/test/scripts/hadoop_escape_chars.bats | 32 +
.../src/test/scripts/hadoop_subcommands.bats | 78 ++
hadoop-common-project/hadoop-kms/pom.xml | 4 +-
.../hadoop-kms/src/main/sbin/kms.sh | 14 +-
hadoop-common-project/hadoop-minikdc/pom.xml | 4 +-
hadoop-common-project/hadoop-nfs/pom.xml | 4 +-
hadoop-common-project/pom.xml | 4 +-
hadoop-dist/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +-
.../hadoop/hdfs/AsyncDistributedFileSystem.java | 59 ++
.../org/apache/hadoop/hdfs/ClientContext.java | 47 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 2 +-
.../hadoop/hdfs/DFSOpsCountStatistics.java | 167 ++++
.../org/apache/hadoop/hdfs/DataStreamer.java | 32 +-
.../hadoop/hdfs/DistributedFileSystem.java | 81 +-
.../apache/hadoop/hdfs/StripedDataStreamer.java | 2 +-
.../hadoop/hdfs/protocol/DatanodeInfo.java | 59 +-
.../ClientNamenodeProtocolTranslatorPB.java | 39 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 8 +
.../delegation/DelegationTokenIdentifier.java | 27 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 51 ++
.../src/main/proto/hdfs.proto | 2 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +-
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 8 +-
.../hadoop-hdfs-native-client/pom.xml | 4 +-
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +-
.../hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 6 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 42 +-
.../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 4 +-
.../hadoop-hdfs/src/main/bin/hdfs | 445 ++++++-----
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +-
.../hdfs/client/CreateEncryptionZoneFlag.java | 71 ++
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 89 ++-
.../apache/hadoop/hdfs/client/package-info.java | 27 +
.../AvailableSpaceBlockPlacementPolicy.java | 11 +-
.../BlockPlacementPolicyDefault.java | 84 +-
.../CacheReplicationMonitor.java | 2 +-
.../hadoop/hdfs/server/common/Storage.java | 65 +-
.../hdfs/server/datanode/DataStorage.java | 2 +-
.../erasurecode/StripedReconstructor.java | 4 +-
.../datanode/metrics/DataNodeMetrics.java | 14 +
.../hadoop/hdfs/server/namenode/NNStorage.java | 25 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 4 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 +-
.../namenode/RedundantEditLogInputStream.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 13 +-
.../apache/hadoop/hdfs/tools/CacheAdmin.java | 6 +-
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 69 +-
.../hdfs/tools/DelegationTokenFetcher.java | 32 +-
.../hadoop/hdfs/tools/erasurecode/ECCli.java | 14 +-
.../StatisticsEditsVisitor.java | 11 +-
.../src/main/shellprofile.d/hadoop-hdfs.sh | 36 +
.../hadoop-hdfs/src/main/shellprofile.d/hdfs.sh | 36 -
.../site/markdown/CentralizedCacheManagement.md | 2 +-
.../src/site/markdown/HDFSCommands.md | 5 +-
.../markdown/HDFSHighAvailabilityWithQJM.md | 2 +-
.../src/site/markdown/TransparentEncryption.md | 13 +-
.../org/apache/hadoop/fs/TestGlobPaths.java | 19 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 278 +++----
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 31 +-
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 32 +-
.../apache/hadoop/hdfs/TestAsyncDFSRename.java | 267 ++++++-
.../apache/hadoop/hdfs/TestDFSPermission.java | 29 +-
.../org/apache/hadoop/hdfs/TestDataStream.java | 85 ++
.../hadoop/hdfs/TestDataTransferProtocol.java | 4 +-
.../hadoop/hdfs/TestDistributedFileSystem.java | 227 +++++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 206 +++--
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 7 +-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 4 +-
.../org/apache/hadoop/hdfs/TestFileAppend.java | 4 +
.../apache/hadoop/hdfs/TestFileCorruption.java | 6 +-
.../java/org/apache/hadoop/hdfs/TestHFlush.java | 10 +-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 4 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 27 +-
.../org/apache/hadoop/hdfs/TestReplication.java | 3 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 12 +-
.../hdfs/security/TestDelegationToken.java | 2 +-
.../blockmanagement/TestBlockManager.java | 4 +-
.../blockmanagement/TestHeartbeatHandling.java | 14 +-
.../TestOverReplicatedBlocks.java | 8 +-
.../server/datanode/BlockReportTestBase.java | 2 +-
.../hdfs/server/datanode/DataNodeTestUtils.java | 119 +--
.../datanode/InternalDataNodeTestUtils.java | 157 ++++
.../TestDataNodeErasureCodingMetrics.java | 153 ++++
.../datanode/TestDataNodeHotSwapVolumes.java | 2 +-
.../datanode/TestDataNodeMetricsLogger.java | 2 +-
.../datanode/TestDataNodeReconfiguration.java | 2 +-
.../TestDataNodeVolumeFailureReporting.java | 36 +
.../hdfs/server/datanode/TestDataStorage.java | 26 +
...TestDnRespectsBlockReportSplitThreshold.java | 6 +-
.../server/datanode/TestFsDatasetCache.java | 2 +-
.../datanode/TestIncrementalBlockReports.java | 2 +-
.../hdfs/server/datanode/TestStorageReport.java | 2 +-
.../server/datanode/TestTriggerBlockReport.java | 2 +-
.../fsdataset/impl/TestFsDatasetImpl.java | 2 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 12 +-
.../hdfs/server/namenode/TestDeleteRace.java | 4 +-
.../server/namenode/TestNameNodeMXBean.java | 24 +-
.../hdfs/server/namenode/TestSaveNamespace.java | 33 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 5 +-
.../namenode/ha/TestPipelinesFailover.java | 6 +-
.../hdfs/tools/TestDelegationTokenFetcher.java | 17 +
.../TestOfflineEditsViewer.java | 30 +
.../apache/hadoop/net/TestNetworkTopology.java | 84 +-
.../src/test/resources/testCryptoConf.xml | 86 ++
.../test/resources/testErasureCodingConf.xml | 16 +
.../scripts/hdfs-functions_test_helper.bash | 58 ++
.../src/test/scripts/hdfs_subcommands.bats | 78 ++
.../hadoop-hdfs/src/test/scripts/run-bats.sh | 43 +
hadoop-hdfs-project/pom.xml | 4 +-
hadoop-mapreduce-project/bin/mapred | 230 +++---
.../hadoop-mapreduce-client-app/pom.xml | 4 +-
.../v2/app/local/LocalContainerAllocator.java | 2 +
.../v2/app/rm/RMContainerAllocator.java | 62 +-
.../v2/app/rm/RMContainerRequestor.java | 6 +
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 3 +-
.../app/local/TestLocalContainerAllocator.java | 43 +
.../v2/app/rm/TestRMContainerAllocator.java | 250 ++++++
.../hadoop-mapreduce-client-common/pom.xml | 4 +-
.../v2/jobhistory/FileNameIndexUtils.java | 2 +-
.../v2/jobhistory/TestFileNameIndexUtils.java | 25 +
.../hadoop-mapreduce-client-core/pom.xml | 4 +-
.../hadoop/mapred/LocatedFileStatusFetcher.java | 4 +-
.../lib/input/CompressedSplitLineReader.java | 5 +
.../org/apache/hadoop/mapreduce/tools/CLI.java | 35 +-
.../src/site/markdown/MapredCommands.md | 3 +-
.../hadoop/mapred/TestLineRecordReader.java | 29 +
.../lib/input/TestLineRecordReader.java | 29 +
.../compressedMultibyteDelimiter.txt.bz2 | Bin 0 -> 1096 bytes
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +-
.../hadoop-mapreduce-client-hs/pom.xml | 8 +-
.../mapreduce/v2/hs/HistoryFileManager.java | 57 +-
.../mapreduce/v2/hs/webapp/HsJobBlock.java | 2 +-
.../v2/hs/webapp/dao/AMAttemptInfo.java | 2 +-
...HistoryFileManagerInitWithNonRunningDFS.java | 75 ++
...estUnnecessaryBlockingOnHistoryFileInfo.java | 323 ++++++++
.../v2/hs/webapp/TestHsWebServicesJobs.java | 12 +-
.../hadoop-mapreduce-client-jobclient/pom.xml | 4 +-
.../hadoop/mapreduce/TestMRJobClient.java | 28 +
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 9 -
.../hadoop-mapreduce-client-nativetask/pom.xml | 4 +-
.../hadoop-mapreduce-client-shuffle/pom.xml | 4 +-
.../hadoop/mapred/FadvisedFileRegion.java | 2 +-
.../hadoop-mapreduce-client/pom.xml | 4 +-
.../hadoop-mapreduce-examples/pom.xml | 4 +-
hadoop-mapreduce-project/pom.xml | 4 +-
.../shellprofile.d/hadoop-mapreduce.sh | 41 +
.../shellprofile.d/mapreduce.sh | 41 -
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-minicluster/pom.xml | 4 +-
hadoop-project-dist/pom.xml | 37 +-
hadoop-project/pom.xml | 16 +-
hadoop-tools/hadoop-ant/pom.xml | 4 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 4 +-
.../main/shellprofile.d/hadoop-archive-logs.sh | 39 +
hadoop-tools/hadoop-archives/pom.xml | 4 +-
.../src/main/shellprofile.d/hadoop-archives.sh | 58 ++
.../hadoop-aws/dev-support/findbugs-exclude.xml | 357 +--------
hadoop-tools/hadoop-aws/pom.xml | 120 ++-
.../apache/hadoop/fs/s3/FileSystemStore.java | 4 +-
.../org/apache/hadoop/fs/s3/S3Credentials.java | 2 +
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 4 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 4 +
.../org/apache/hadoop/fs/s3a/Constants.java | 20 +-
.../hadoop/fs/s3a/S3AFastOutputStream.java | 11 +-
.../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 12 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 484 +++++++-----
.../apache/hadoop/fs/s3a/S3AInputStream.java | 342 ++++++--
.../hadoop/fs/s3a/S3AInstrumentation.java | 457 +++++++++++
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 40 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 167 +++-
.../hadoop/fs/contract/s3/S3Contract.java | 10 +
.../fs/contract/s3/TestS3ContractRootDir.java | 15 +
.../fs/contract/s3/TestS3ContractSeek.java | 9 +
.../hadoop/fs/contract/s3a/S3AContract.java | 9 +-
.../fs/contract/s3a/TestS3AContractRootDir.java | 39 +-
.../fs/contract/s3n/NativeS3Contract.java | 9 +-
.../apache/hadoop/fs/s3/TestS3Credentials.java | 3 +
.../hadoop/fs/s3a/TestS3AConfiguration.java | 71 +-
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 195 ++++-
.../fs/s3a/scale/TestS3ADeleteManyFiles.java | 1 -
.../scale/TestS3AInputStreamPerformance.java | 285 +++++++
.../Jets3tNativeS3FileSystemContractTest.java | 30 -
.../TestJets3tNativeS3FileSystemContract.java | 30 +
.../src/test/resources/log4j.properties | 3 +
hadoop-tools/hadoop-azure/pom.xml | 2 +-
hadoop-tools/hadoop-datajoin/pom.xml | 4 +-
hadoop-tools/hadoop-distcp/pom.xml | 4 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 32 +-
.../org/apache/hadoop/tools/OptionsParser.java | 49 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 16 +-
.../src/main/shellprofile.d/hadoop-distcp.sh | 62 ++
.../apache/hadoop/tools/TestDistCpViewFs.java | 93 +--
.../apache/hadoop/tools/TestOptionsParser.java | 19 +-
.../hadoop/tools/mapred/TestCopyMapper.java | 85 ++
hadoop-tools/hadoop-extras/pom.xml | 4 +-
.../src/main/shellprofile.d/hadoop-extras.sh | 39 +
hadoop-tools/hadoop-gridmix/pom.xml | 4 +-
.../src/main/shellprofile.d/hadoop-gridmix.sh | 36 +
.../src/site/markdown/GridMix.md.vm | 18 +-
hadoop-tools/hadoop-kafka/pom.xml | 4 +-
hadoop-tools/hadoop-openstack/pom.xml | 26 +-
hadoop-tools/hadoop-pipes/pom.xml | 4 +-
hadoop-tools/hadoop-rumen/pom.xml | 4 +-
.../src/main/shellprofile.d/hadoop-rumen.sh | 58 ++
.../hadoop-rumen/src/site/markdown/Rumen.md.vm | 40 +-
hadoop-tools/hadoop-sls/pom.xml | 4 +-
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 18 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 9 +
hadoop-tools/hadoop-streaming/pom.xml | 4 +-
.../apache/hadoop/streaming/DumpTypedBytes.java | 3 +-
.../hadoop/streaming/HadoopStreaming.java | 3 +-
.../apache/hadoop/streaming/LoadTypedBytes.java | 3 +-
.../src/main/shellprofile.d/hadoop-streaming.sh | 55 ++
.../src/site/markdown/HadoopStreaming.md.vm | 30 +-
hadoop-tools/hadoop-tools-dist/pom.xml | 4 +-
hadoop-tools/pom.xml | 4 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 394 ++++++----
.../dev-support/findbugs-exclude.xml | 2 +-
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +-
.../hadoop/yarn/api/ApplicationConstants.java | 7 +
.../hadoop/yarn/conf/YarnConfiguration.java | 67 +-
.../yarn/conf/TestYarnConfigurationFields.java | 14 +-
.../pom.xml | 4 +-
.../pom.xml | 4 +-
.../hadoop-yarn-applications/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 10 -
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 787 ++++++++++++-------
.../yarn/client/api/impl/TestAMRMClient.java | 13 +-
.../yarn/client/api/impl/TestAMRMProxy.java | 6 +-
.../api/impl/TestDistributedScheduling.java | 307 ++++++++
.../hadoop/yarn/client/cli/TestLogsCLI.java | 495 +++++++++++-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 +-
.../logaggregation/AggregatedLogFormat.java | 20 +
.../yarn/logaggregation/LogCLIHelpers.java | 396 +++++++---
.../src/main/resources/yarn-default.xml | 13 +
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 +-
.../pom.xml | 4 +-
...pplicationHistoryManagerOnTimelineStore.java | 43 +-
.../webapp/AHSWebServices.java | 266 ++++++-
...pplicationHistoryManagerOnTimelineStore.java | 41 +-
.../webapp/TestAHSWebServices.java | 203 ++++-
.../hadoop-yarn-server-common/pom.xml | 4 +-
.../protocolrecords/NodeHeartbeatResponse.java | 8 +
.../RegisterNodeManagerResponse.java | 6 +-
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 88 ++-
.../pb/RegisterNodeManagerResponsePBImpl.java | 46 +-
.../api/records/ContainerQueuingLimit.java | 44 ++
.../api/records/QueuedContainersStatus.java | 2 +-
.../impl/pb/ContainerQueuingLimitPBImpl.java | 80 ++
.../api/records/impl/pb/NodeStatusPBImpl.java | 14 +-
.../impl/pb/QueuedContainersStatusPBImpl.java | 6 +-
.../metrics/ApplicationMetricsConstants.java | 3 +
.../yarn/server/webapp/dao/ContainerInfo.java | 6 +
.../yarn_server_common_service_protos.proto | 8 +
.../hadoop-yarn-server-nodemanager/pom.xml | 4 +-
.../hadoop/yarn/server/nodemanager/Context.java | 4 +-
.../yarn/server/nodemanager/NodeManager.java | 8 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 258 ++++--
.../containermanager/AuxServices.java | 43 +-
.../AuxiliaryServiceWithCustomClassLoader.java | 201 +++++
.../containermanager/ContainerManager.java | 42 +
.../containermanager/ContainerManagerImpl.java | 43 +-
.../application/ApplicationImpl.java | 8 +-
.../runtime/DockerLinuxContainerRuntime.java | 47 ++
.../logaggregation/AppLogAggregatorImpl.java | 4 +
.../logaggregation/LogAggregationService.java | 12 +-
.../loghandler/NonAggregatingLogHandler.java | 4 +
.../queuing/QueuingContainerManagerImpl.java | 96 ++-
.../recovery/NMLeveldbStateStoreService.java | 26 +-
.../recovery/NMNullStateStoreService.java | 4 +
.../recovery/NMStateStoreService.java | 9 +
.../nodemanager/scheduler/LocalScheduler.java | 6 +-
.../OpportunisticContainerAllocator.java | 17 +-
.../nodemanager/webapp/NMWebServices.java | 22 +-
.../nodemanager/TestNodeStatusUpdater.java | 121 +++
.../amrmproxy/BaseAMRMProxyTest.java | 3 +-
.../containermanager/TestAuxServices.java | 119 ++-
.../TestContainerManagerRecovery.java | 84 ++
.../runtime/TestDockerContainerRuntime.java | 115 +++
.../TestLogAggregationService.java | 11 +-
.../TestNonAggregatingLogHandler.java | 58 +-
.../monitor/TestContainerMetrics.java | 11 -
.../recovery/NMMemoryStateStoreService.java | 6 +
.../TestNMLeveldbStateStoreService.java | 12 +
.../scheduler/TestLocalScheduler.java | 9 +-
.../nodemanager/webapp/TestNMWebServices.java | 12 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 +-
.../server/resourcemanager/ClientRMService.java | 11 +-
.../server/resourcemanager/ClusterMonitor.java | 6 +-
.../DistributedSchedulingService.java | 194 +++--
.../resourcemanager/NodesListManager.java | 120 ++-
.../yarn/server/resourcemanager/RMContext.java | 4 +
.../server/resourcemanager/RMContextImpl.java | 14 +
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../server/resourcemanager/ResourceManager.java | 2 +
.../resourcemanager/ResourceTrackerService.java | 70 +-
.../resourcemanager/amlauncher/AMLauncher.java | 14 +
.../metrics/ApplicaitonStateUpdatedEvent.java | 47 ++
.../metrics/ContainerFinishedEvent.java | 9 +-
.../metrics/SystemMetricsEventType.java | 1 +
.../metrics/SystemMetricsPublisher.java | 97 ++-
.../monitor/SchedulingMonitor.java | 16 +-
.../CapacitySchedulerPreemptionUtils.java | 35 +-
.../capacity/FifoCandidatesSelector.java | 26 +-
.../capacity/PreemptableResourceCalculator.java | 139 +++-
.../ProportionalCapacityPreemptionPolicy.java | 96 ++-
.../ReservedContainerCandidatesSelector.java | 316 ++++++++
.../monitor/capacity/TempQueuePerPartition.java | 174 +++-
.../server/resourcemanager/rmapp/RMAppImpl.java | 19 +-
.../rmcontainer/RMContainer.java | 2 +
.../rmcontainer/RMContainerImpl.java | 10 +
.../server/resourcemanager/rmnode/RMNode.java | 5 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 53 +-
.../scheduler/AbstractYarnScheduler.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 4 +
.../scheduler/SchedulerNode.java | 29 +-
.../scheduler/capacity/CapacityScheduler.java | 10 +-
.../CapacitySchedulerConfiguration.java | 24 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 1 +
.../common/fica/FiCaSchedulerNode.java | 7 +-
.../distributed/NodeQueueLoadMonitor.java | 273 +++++++
.../distributed/QueueLimitCalculator.java | 125 +++
.../scheduler/distributed/TopKNodeSelector.java | 223 ------
.../scheduler/fair/FSAppAttempt.java | 1 +
.../scheduler/fair/FairScheduler.java | 3 +-
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../security/DelegationTokenRenewer.java | 9 +-
.../security/QueueACLsManager.java | 32 +-
.../webapp/FairSchedulerPage.java | 1 +
.../resourcemanager/webapp/RMWebServices.java | 30 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 9 +
.../yarn/server/resourcemanager/MockNM.java | 21 +-
.../yarn/server/resourcemanager/MockNodes.java | 9 +
.../resourcemanager/TestApplicationACLs.java | 5 +-
.../TestApplicationMasterLauncher.java | 5 +
.../resourcemanager/TestClientRMService.java | 15 +-
.../TestDistributedSchedulingService.java | 68 +-
.../resourcemanager/TestRMAdminService.java | 99 +++
.../TestResourceTrackerService.java | 483 +++++++++++-
.../TestWorkPreservingRMRestart.java | 72 +-
.../metrics/TestSystemMetricsPublisher.java | 59 +-
...alCapacityPreemptionPolicyMockFramework.java | 689 ++++++++++++++++
...estProportionalCapacityPreemptionPolicy.java | 62 +-
...pacityPreemptionPolicyForNodePartitions.java | 687 +---------------
...tyPreemptionPolicyForReservedContainers.java | 430 ++++++++++
...alCapacityPreemptionPolicyMockFramework.java | 247 ++++++
.../TestSchedulerApplicationAttempt.java | 3 +-
.../CapacitySchedulerPreemptionTestBase.java | 149 ++++
.../capacity/TestCapacityScheduler.java | 2 +-
.../TestCapacitySchedulerLazyPreemption.java | 638 +++++++++++++++
.../TestCapacitySchedulerPreemption.java | 677 ----------------
...TestCapacitySchedulerSurgicalPreemption.java | 246 ++++++
.../distributed/TestNodeQueueLoadMonitor.java | 195 +++++
.../distributed/TestTopKNodeSelector.java | 147 ----
.../scheduler/fair/TestFairScheduler.java | 4 +-
.../scheduler/fifo/TestFifoScheduler.java | 7 +-
.../webapp/TestRMWebServicesNodeLabels.java | 107 +++
.../webapp/TestRMWebServicesNodes.java | 12 +-
.../pom.xml | 4 +-
.../hadoop-yarn-server-tests/pom.xml | 4 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 67 +-
.../pom.xml | 4 +-
.../hadoop-yarn-server-web-proxy/pom.xml | 4 +-
.../server/webproxy/WebAppProxyServlet.java | 11 +-
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 +-
.../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +-
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 +-
.../hadoop-yarn/shellprofile.d/hadoop-yarn.sh | 62 ++
.../hadoop-yarn/shellprofile.d/yarn.sh | 62 --
hadoop-yarn-project/pom.xml | 4 +-
pom.xml | 5 +-
466 files changed, 20139 insertions(+), 6382 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76279f58/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76279f58/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76279f58/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index d0e665d,2a29d17..da4be41
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@@ -58,7 -61,194 +63,200 @@@ function hadoop_usag
hadoop_add_subcommand "storagepolicies" "list/get/set block storage policies"
hadoop_add_subcommand "version" "print the version"
hadoop_add_subcommand "zkfc" "run the ZK Failover Controller daemon"
- hadoop_generate_usage "${MYNAME}" false
+ hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
+ }
+
+ ## @description Default command handler for hadoop command
+ ## @audience public
+ ## @stability stable
+ ## @replaceable no
+ ## @param CLI arguments
+ function hdfscmd_case
+ {
+ subcmd=$1
+ shift
+
+ case ${subcmd} in
+ balancer)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.balancer.Balancer
+ hadoop_debug "Appending HADOOP_BALANCER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}"
+ ;;
+ cacheadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin
+ ;;
+ classpath)
+ hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
+ ;;
+ crypto)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CryptoAdmin
+ ;;
+ datanode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ # Determine if we're starting a secure datanode, and
+ # if so, redefine appropriate variables
+ if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
+ HADOOP_SUBCMD_SECURESERVICE="true"
+ HADOOP_SUBCMD_SECUREUSER="${HADOOP_SECURE_DN_USER}"
+
+ # backward compatiblity
+ HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}"
+ HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}"
+
+ hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
+ hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}"
+ HADOOP_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
+ else
+ hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
+ fi
+ ;;
+ debug)
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DebugAdmin'
+ ;;
+ dfs)
+ HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ dfsadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSAdmin
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ envvars)
+ echo "JAVA_HOME='${JAVA_HOME}'"
+ echo "HADOOP_HDFS_HOME='${HADOOP_HDFS_HOME}'"
+ echo "HDFS_DIR='${HDFS_DIR}'"
+ echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'"
+ echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
+ echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
+ echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
+ echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
+ exit 0
+ ;;
+ erasurecode)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ fetchdt)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
+ ;;
+ fsck)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ getconf)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetConf
+ ;;
+ groups)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetGroups
+ ;;
+ haadmin)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSHAAdmin
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
+ journalnode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
+ hadoop_debug "Appending HADOOP_JOURNALNODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}"
+ ;;
+ jmxget)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
+ ;;
+ lsSnapshottableDir)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
+ ;;
+ mover)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.mover.Mover
+ hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}"
+ ;;
+ namenode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.NameNode'
+ hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}"
+ hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
+ ;;
+ nfs3)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then
+ HADOOP_SUBCMD_SECURESERVICE="true"
+ HADOOP_SUBCMD_SECUREUSER="${HADOOP_PRIVILEGED_NFS_USER}"
+
+ # backward compatiblity
+ HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}"
+ HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}"
+
+ hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
+ hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter
+ else
+ hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}"
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3
+ fi
+ ;;
+ oev)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
+ ;;
+ oiv)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
+ ;;
+ oiv_legacy)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
+ ;;
+ portmap)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap
+ hadoop_debug "Appending HADOOP_PORTMAP_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}"
+ ;;
++ scm)
++ supportdaemonization="true"
++ CLASS='org.apache.hadoop.ozone.storage.StorageContainerManager'
++ hadoop_debug "Appending HADOOP_SCM_OPTS onto HADOOP_OPTS"
++ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SCM_OPTS}"
++ ;;
+ secondarynamenode)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
+ hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}"
+ hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
+ ;;
+ snapshotDiff)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
+ ;;
+ storagepolicies)
+ HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
+ ;;
+ version)
+ HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
+ ;;
+ zkfc)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
+ hadoop_debug "Appending HADOOP_ZKFC_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}"
+ ;;
+ *)
+ HADOOP_CLASSNAME="${subcmd}"
+ if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
+ hadoop_exit_with_usage 1
+ fi
+ ;;
+ esac
}
# let's locate libexec...
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76279f58/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDFS-8449. Add tasks count metrics
to datanode for ECWorker. Contributed by Bo Li.
Posted by ar...@apache.org.
HDFS-8449. Add tasks count metrics to datanode for ECWorker. Contributed by Bo Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad944112
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad944112
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad944112
Branch: refs/heads/HDFS-7240
Commit: ad9441122f31547fcab29f50e64d52a8895906b6
Parents: 4578820
Author: Kai Zheng <ka...@intel.com>
Authored: Sun May 15 23:39:09 2016 -0700
Committer: Kai Zheng <ka...@intel.com>
Committed: Sun May 15 23:39:09 2016 -0700
----------------------------------------------------------------------
.../erasurecode/StripedReconstructor.java | 4 +-
.../datanode/metrics/DataNodeMetrics.java | 14 ++
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 32 +++-
.../hadoop/hdfs/TestReconstructStripedFile.java | 27 +---
.../TestDataNodeErasureCodingMetrics.java | 153 +++++++++++++++++++
5 files changed, 204 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad944112/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
index 1b59b22..c80bf96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
@@ -179,11 +179,11 @@ class StripedReconstructor implements Runnable {
// block replication.
} catch (Throwable e) {
LOG.warn("Failed to reconstruct striped block: {}", blockGroup, e);
+ datanode.getMetrics().incrECFailedReconstructionTasks();
} finally {
datanode.decrementXmitsInProgress();
-
+ datanode.getMetrics().incrECReconstructionTasks();
stripedReader.close();
-
stripedWriter.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad944112/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 085762b..3d504d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -126,6 +126,11 @@ public class DataNodeMetrics {
@Metric MutableRate sendDataPacketTransferNanos;
final MutableQuantiles[] sendDataPacketTransferNanosQuantiles;
+ @Metric("Count of erasure coding reconstruction tasks")
+ MutableCounterLong ecReconstructionTasks;
+ @Metric("Count of erasure coding failed reconstruction tasks")
+ MutableCounterLong ecFailedReconstructionTasks;
+
final MetricsRegistry registry = new MetricsRegistry("datanode");
final String name;
JvmMetrics jvmMetrics = null;
@@ -415,4 +420,13 @@ public class DataNodeMetrics {
q.add(latencyMs);
}
}
+
+ public void incrECReconstructionTasks() {
+ ecReconstructionTasks.incr();
+ }
+
+ public void incrECFailedReconstructionTasks() {
+ ecFailedReconstructionTasks.incr();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad944112/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
index 3c58133..6dcccc3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
@@ -74,7 +74,7 @@ public class StripedFileTestUtil {
static int numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 2;
static int BLOCK_GROUP_SIZE = blockSize * NUM_DATA_BLOCKS;
- static byte[] generateBytes(int cnt) {
+ public static byte[] generateBytes(int cnt) {
byte[] bytes = new byte[cnt];
for (int i = 0; i < cnt; i++) {
bytes[i] = getByte(i);
@@ -502,4 +502,34 @@ public class StripedFileTestUtil {
}
}
}
+
+ /**
+ * Wait for the reconstruction to be finished when the file has
+ * corrupted blocks.
+ */
+ public static LocatedBlocks waitForReconstructionFinished(Path file,
+ DistributedFileSystem fs, int groupSize)
+ throws Exception {
+ final int attempts = 60;
+ for (int i = 0; i < attempts; i++) {
+ LocatedBlocks locatedBlocks = getLocatedBlocks(file, fs);
+ LocatedStripedBlock lastBlock =
+ (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
+ DatanodeInfo[] storageInfos = lastBlock.getLocations();
+ if (storageInfos.length >= groupSize) {
+ return locatedBlocks;
+ }
+ Thread.sleep(1000);
+ }
+ throw new IOException("Time out waiting for EC block reconstruction.");
+ }
+
+ /**
+ * Get the located blocks of a file.
+ */
+ public static LocatedBlocks getLocatedBlocks(Path file,
+ DistributedFileSystem fs)
+ throws IOException {
+ return fs.getClient().getLocatedBlocks(file.toString(), 0, Long.MAX_VALUE);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad944112/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index 7155e74..36d2dbd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -269,7 +269,8 @@ public class TestReconstructStripedFile {
DFSTestUtil.writeFile(fs, file, data);
StripedFileTestUtil.waitBlockGroupsReported(fs, fileName);
- LocatedBlocks locatedBlocks = getLocatedBlocks(file);
+ LocatedBlocks locatedBlocks =
+ StripedFileTestUtil.getLocatedBlocks(file, fs);
assertEquals(locatedBlocks.getFileLength(), fileLen);
LocatedStripedBlock lastBlock =
@@ -325,7 +326,7 @@ public class TestReconstructStripedFile {
int stoppedDN = generateErrors(errorMap, type);
// Check the locatedBlocks of the file again
- locatedBlocks = getLocatedBlocks(file);
+ locatedBlocks = StripedFileTestUtil.getLocatedBlocks(file, fs);
lastBlock = (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
storageInfos = lastBlock.getLocations();
assertEquals(storageInfos.length, groupSize - stoppedDN);
@@ -338,7 +339,7 @@ public class TestReconstructStripedFile {
}
}
- waitForReconstructionFinished(file, groupSize);
+ StripedFileTestUtil.waitForReconstructionFinished(file, fs, groupSize);
targetDNs = sortTargetsByReplicas(blocks, targetDNs);
@@ -381,26 +382,6 @@ public class TestReconstructStripedFile {
return result;
}
- private LocatedBlocks waitForReconstructionFinished(Path file, int groupSize)
- throws Exception {
- final int ATTEMPTS = 60;
- for (int i = 0; i < ATTEMPTS; i++) {
- LocatedBlocks locatedBlocks = getLocatedBlocks(file);
- LocatedStripedBlock lastBlock =
- (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
- DatanodeInfo[] storageInfos = lastBlock.getLocations();
- if (storageInfos.length >= groupSize) {
- return locatedBlocks;
- }
- Thread.sleep(1000);
- }
- throw new IOException ("Time out waiting for EC block reconstruction.");
- }
-
- private LocatedBlocks getLocatedBlocks(Path file) throws IOException {
- return fs.getClient().getLocatedBlocks(file.toString(), 0, Long.MAX_VALUE);
- }
-
/*
* Tests that processErasureCodingTasks should not throw exceptions out due to
* invalid ECTask submission.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad944112/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
new file mode 100644
index 0000000..e401fed
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.StripedFileTestUtil;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assert.assertEquals;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+
+/**
+ * This file tests the erasure coding metrics in DataNode.
+ */
+public class TestDataNodeErasureCodingMetrics {
+ public static final Log LOG = LogFactory.
+ getLog(TestDataNodeErasureCodingMetrics.class);
+
+ private static final int DATA_BLK_NUM = StripedFileTestUtil.NUM_DATA_BLOCKS;
+ private static final int PARITY_BLK_NUM =
+ StripedFileTestUtil.NUM_PARITY_BLOCKS;
+ private static final int CELLSIZE =
+ StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
+ private static final int BLOCKSIZE = CELLSIZE;
+ private static final int GROUPSIZE = DATA_BLK_NUM + PARITY_BLK_NUM;
+ private static final int DN_NUM = GROUPSIZE + 1;
+
+ private MiniDFSCluster cluster;
+ private Configuration conf;
+ private DistributedFileSystem fs;
+
+ @Before
+ public void setup() throws IOException {
+ conf = new Configuration();
+
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(DN_NUM).build();
+ cluster.waitActive();
+ cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout = 120000)
+ public void testEcTasks() throws Exception {
+ DataNode workerDn = doTest("/testEcTasks");
+ MetricsRecordBuilder rb = getMetrics(workerDn.getMetrics().name());
+ assertCounter("EcReconstructionTasks", (long) 1, rb);
+ assertCounter("EcFailedReconstructionTasks", (long) 0, rb);
+ }
+
+ private DataNode doTest(String fileName) throws Exception {
+
+ Path file = new Path(fileName);
+ long fileLen = DATA_BLK_NUM * BLOCKSIZE;
+ final byte[] data = StripedFileTestUtil.generateBytes((int) fileLen);
+ DFSTestUtil.writeFile(fs, file, data);
+ StripedFileTestUtil.waitBlockGroupsReported(fs, fileName);
+
+ LocatedBlocks locatedBlocks =
+ StripedFileTestUtil.getLocatedBlocks(file, fs);
+ //only one block group
+ LocatedStripedBlock lastBlock =
+ (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
+ DataNode workerDn = null;
+ DatanodeInfo[] locations = lastBlock.getLocations();
+ assertEquals(locations.length, GROUPSIZE);
+ // we have ONE extra datanode in addition to the GROUPSIZE datanodes, here
+ // is to find the extra datanode that the reconstruction task will run on,
+ // according to the current block placement logic for striped files.
+ // This can be improved later to be flexible regardless wherever the task
+ // runs.
+ for (DataNode dn: cluster.getDataNodes()) {
+ boolean appear = false;
+ for (DatanodeInfo info: locations) {
+ if (dn.getDatanodeUuid().equals(info.getDatanodeUuid())) {
+ appear = true;
+ break;
+ }
+ }
+ if(!appear) {
+ workerDn = dn;
+ break;
+ }
+ }
+ byte[] indices = lastBlock.getBlockIndices();
+ //corrupt the first block
+ DataNode toCorruptDn = cluster.getDataNodes().get(indices[0]);
+ toCorruptDn.shutdown();
+ setDataNodeDead(toCorruptDn.getDatanodeId());
+ DFSTestUtil.waitForDatanodeState(cluster, toCorruptDn.getDatanodeUuid(),
+ false, 10000 );
+ final BlockManager bm = cluster.getNamesystem().getBlockManager();
+ BlockManagerTestUtil.getComputedDatanodeWork(bm);
+ cluster.triggerHeartbeats();
+ StripedFileTestUtil.waitForReconstructionFinished(file, fs, GROUPSIZE);
+
+ return workerDn;
+ }
+
+ private void setDataNodeDead(DatanodeID dnID) throws IOException {
+ DatanodeDescriptor dnd =
+ NameNodeAdapter.getDatanode(cluster.getNamesystem(), dnID);
+ DFSTestUtil.setDatanodeDead(dnd);
+ BlockManagerTestUtil.checkHeartbeat(
+ cluster.getNamesystem().getBlockManager());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HADOOP-12782. Faster LDAP group
name resolution with ActiveDirectory. Contributed by Wei-Chiu Chuang
Posted by ar...@apache.org.
HADOOP-12782. Faster LDAP group name resolution with ActiveDirectory. Contributed by Wei-Chiu Chuang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/182fc198
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/182fc198
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/182fc198
Branch: refs/heads/HDFS-7240
Commit: 182fc1986a984ed0be6bed297390a830c2305af1
Parents: d4274c6
Author: Kai Zheng <ka...@intel.com>
Authored: Thu May 19 07:15:52 2016 -0700
Committer: Kai Zheng <ka...@intel.com>
Committed: Thu May 19 07:15:52 2016 -0700
----------------------------------------------------------------------
.../hadoop/security/LdapGroupsMapping.java | 228 +++++++++++++++----
.../src/main/resources/core-default.xml | 12 +
.../src/site/markdown/GroupsMapping.md | 6 +
.../hadoop/security/TestLdapGroupsMapping.java | 31 +--
.../security/TestLdapGroupsMappingBase.java | 71 ++++--
.../TestLdapGroupsMappingWithOneQuery.java | 100 ++++++++
.../TestLdapGroupsMappingWithPosixGroup.java | 41 ++--
7 files changed, 393 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/182fc198/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index d72aa1e..498b92e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -34,6 +34,8 @@ import javax.naming.directory.DirContext;
import javax.naming.directory.InitialDirContext;
import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
+import javax.naming.ldap.LdapName;
+import javax.naming.ldap.Rdn;
import org.apache.commons.io.Charsets;
import org.apache.commons.logging.Log;
@@ -136,6 +138,13 @@ public class LdapGroupsMapping
public static final String GROUP_SEARCH_FILTER_DEFAULT = "(objectClass=group)";
/*
+ * LDAP attribute to use for determining group membership
+ */
+ public static final String MEMBEROF_ATTR_KEY =
+ LDAP_CONFIG_PREFIX + ".search.attr.memberof";
+ public static final String MEMBEROF_ATTR_DEFAULT = "";
+
+ /*
* LDAP attribute to use for determining group membership
*/
public static final String GROUP_MEMBERSHIP_ATTR_KEY = LDAP_CONFIG_PREFIX + ".search.attr.member";
@@ -189,11 +198,13 @@ public class LdapGroupsMapping
private String baseDN;
private String groupSearchFilter;
private String userSearchFilter;
+ private String memberOfAttr;
private String groupMemberAttr;
private String groupNameAttr;
private String posixUidAttr;
private String posixGidAttr;
private boolean isPosix;
+ private boolean useOneQuery;
public static final int RECONNECT_RETRY_COUNT = 3;
@@ -229,58 +240,173 @@ public class LdapGroupsMapping
return Collections.emptyList();
}
-
- List<String> doGetGroups(String user) throws NamingException {
- List<String> groups = new ArrayList<String>();
- DirContext ctx = getDirContext();
+ /**
+ * A helper method to get the Relative Distinguished Name (RDN) from
+ * Distinguished name (DN). According to Active Directory documentation,
+ * a group object's RDN is a CN.
+ *
+ * @param distinguishedName A string representing a distinguished name.
+ * @throws NamingException if the DN is malformed.
+ * @return a string which represents the RDN
+ */
+ private String getRelativeDistinguishedName(String distinguishedName)
+ throws NamingException {
+ LdapName ldn = new LdapName(distinguishedName);
+ List<Rdn> rdns = ldn.getRdns();
+ if (rdns.isEmpty()) {
+ throw new NamingException("DN is empty");
+ }
+ Rdn rdn = rdns.get(rdns.size()-1);
+ if (rdn.getType().equalsIgnoreCase(groupNameAttr)) {
+ String groupName = (String)rdn.getValue();
+ return groupName;
+ }
+ throw new NamingException("Unable to find RDN: The DN " +
+ distinguishedName + " is malformed.");
+ }
- // Search for the user. We'll only ever need to look at the first result
- NamingEnumeration<SearchResult> results = ctx.search(baseDN,
- userSearchFilter,
- new Object[]{user},
- SEARCH_CONTROLS);
- if (results.hasMoreElements()) {
- SearchResult result = results.nextElement();
- String userDn = result.getNameInNamespace();
+ /**
+ * Look up groups using posixGroups semantics. Use posix gid/uid to find
+ * groups of the user.
+ *
+ * @param result the result object returned from the prior user lookup.
+ * @param c the context object of the LDAP connection.
+ * @return an object representing the search result.
+ *
+ * @throws NamingException if the server does not support posixGroups
+ * semantics.
+ */
+ private NamingEnumeration<SearchResult> lookupPosixGroup(SearchResult result,
+ DirContext c) throws NamingException {
+ String gidNumber = null;
+ String uidNumber = null;
+ Attribute gidAttribute = result.getAttributes().get(posixGidAttr);
+ Attribute uidAttribute = result.getAttributes().get(posixUidAttr);
+ String reason = "";
+ if (gidAttribute == null) {
+ reason = "Can't find attribute '" + posixGidAttr + "'.";
+ } else {
+ gidNumber = gidAttribute.get().toString();
+ }
+ if (uidAttribute == null) {
+ reason = "Can't find attribute '" + posixUidAttr + "'.";
+ } else {
+ uidNumber = uidAttribute.get().toString();
+ }
+ if (uidNumber != null && gidNumber != null) {
+ return c.search(baseDN,
+ "(&"+ groupSearchFilter + "(|(" + posixGidAttr + "={0})" +
+ "(" + groupMemberAttr + "={1})))",
+ new Object[] {gidNumber, uidNumber},
+ SEARCH_CONTROLS);
+ }
+ throw new NamingException("The server does not support posixGroups " +
+ "semantics. Reason: " + reason +
+ " Returned user object: " + result.toString());
+ }
- NamingEnumeration<SearchResult> groupResults = null;
+ /**
+ * Perform the second query to get the groups of the user.
+ *
+ * If posixGroups is enabled, use use posix gid/uid to find.
+ * Otherwise, use the general group member attribute to find it.
+ *
+ * @param result the result object returned from the prior user lookup.
+ * @param c the context object of the LDAP connection.
+ * @return a list of strings representing group names of the user.
+ * @throws NamingException if unable to find group names
+ */
+ private List<String> lookupGroup(SearchResult result, DirContext c)
+ throws NamingException {
+ List<String> groups = new ArrayList<String>();
- if (isPosix) {
- String gidNumber = null;
- String uidNumber = null;
- Attribute gidAttribute = result.getAttributes().get(posixGidAttr);
- Attribute uidAttribute = result.getAttributes().get(posixUidAttr);
- if (gidAttribute != null) {
- gidNumber = gidAttribute.get().toString();
- }
- if (uidAttribute != null) {
- uidNumber = uidAttribute.get().toString();
- }
- if (uidNumber != null && gidNumber != null) {
- groupResults =
- ctx.search(baseDN,
- "(&"+ groupSearchFilter + "(|(" + posixGidAttr + "={0})" +
- "(" + groupMemberAttr + "={1})))",
- new Object[] { gidNumber, uidNumber },
- SEARCH_CONTROLS);
+ NamingEnumeration<SearchResult> groupResults = null;
+ // perform the second LDAP query
+ if (isPosix) {
+ groupResults = lookupPosixGroup(result, c);
+ } else {
+ String userDn = result.getNameInNamespace();
+ groupResults =
+ c.search(baseDN,
+ "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))",
+ new Object[]{userDn},
+ SEARCH_CONTROLS);
+ }
+ // if the second query is successful, group objects of the user will be
+ // returned. Get group names from the returned objects.
+ if (groupResults != null) {
+ while (groupResults.hasMoreElements()) {
+ SearchResult groupResult = groupResults.nextElement();
+ Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
+ if (groupName == null) {
+ throw new NamingException("The group object does not have " +
+ "attribute '" + groupNameAttr + "'.");
}
- } else {
- groupResults =
- ctx.search(baseDN,
- "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))",
- new Object[]{userDn},
- SEARCH_CONTROLS);
+ groups.add(groupName.get().toString());
}
- if (groupResults != null) {
- while (groupResults.hasMoreElements()) {
- SearchResult groupResult = groupResults.nextElement();
- Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
- groups.add(groupName.get().toString());
- }
+ }
+ return groups;
+ }
+
+ /**
+ * Perform LDAP queries to get group names of a user.
+ *
+ * Perform the first LDAP query to get the user object using the user's name.
+ * If one-query is enabled, retrieve the group names from the user object.
+ * If one-query is disabled, or if it failed, perform the second query to
+ * get the groups.
+ *
+ * @param user user name
+ * @return a list of group names for the user. If the user can not be found,
+ * return an empty string array.
+ * @throws NamingException if unable to get group names
+ */
+ List<String> doGetGroups(String user) throws NamingException {
+ DirContext c = getDirContext();
+
+ // Search for the user. We'll only ever need to look at the first result
+ NamingEnumeration<SearchResult> results = c.search(baseDN,
+ userSearchFilter, new Object[]{user}, SEARCH_CONTROLS);
+ // return empty list if the user can not be found.
+ if (!results.hasMoreElements()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("doGetGroups(" + user + ") return no groups because the " +
+ "user is not found.");
}
+ return new ArrayList<String>();
}
+ SearchResult result = results.nextElement();
+ List<String> groups = null;
+ if (useOneQuery) {
+ try {
+ /**
+ * For Active Directory servers, the user object has an attribute
+ * 'memberOf' that represents the DNs of group objects to which the
+ * user belongs. So the second query may be skipped.
+ */
+ Attribute groupDNAttr = result.getAttributes().get(memberOfAttr);
+ if (groupDNAttr == null) {
+ throw new NamingException("The user object does not have '" +
+ memberOfAttr + "' attribute." +
+ "Returned user object: " + result.toString());
+ }
+ groups = new ArrayList<String>();
+ NamingEnumeration groupEnumeration = groupDNAttr.getAll();
+ while (groupEnumeration.hasMore()) {
+ String groupDN = groupEnumeration.next().toString();
+ groups.add(getRelativeDistinguishedName(groupDN));
+ }
+ } catch (NamingException e) {
+ // If the first lookup failed, fall back to the typical scenario.
+ LOG.info("Failed to get groups from the first lookup. Initiating " +
+ "the second LDAP query using the user's DN.", e);
+ }
+ }
+ if (groups == null || groups.isEmpty()) {
+ groups = lookupGroup(result, c);
+ }
if (LOG.isDebugEnabled()) {
LOG.debug("doGetGroups(" + user + ") return " + groups);
}
@@ -366,6 +492,11 @@ public class LdapGroupsMapping
conf.get(USER_SEARCH_FILTER_KEY, USER_SEARCH_FILTER_DEFAULT);
isPosix = groupSearchFilter.contains(POSIX_GROUP) && userSearchFilter
.contains(POSIX_ACCOUNT);
+ memberOfAttr =
+ conf.get(MEMBEROF_ATTR_KEY, MEMBEROF_ATTR_DEFAULT);
+ // if memberOf attribute is set, resolve group names from the attribute
+ // of user objects.
+ useOneQuery = !memberOfAttr.isEmpty();
groupMemberAttr =
conf.get(GROUP_MEMBERSHIP_ATTR_KEY, GROUP_MEMBERSHIP_ATTR_DEFAULT);
groupNameAttr =
@@ -379,8 +510,15 @@ public class LdapGroupsMapping
SEARCH_CONTROLS.setTimeLimit(dirSearchTimeout);
// Limit the attributes returned to only those required to speed up the search.
// See HADOOP-10626 and HADOOP-12001 for more details.
- SEARCH_CONTROLS.setReturningAttributes(
- new String[] {groupNameAttr, posixUidAttr, posixGidAttr});
+ String[] returningAttributes;
+ if (useOneQuery) {
+ returningAttributes = new String[] {
+ groupNameAttr, posixUidAttr, posixGidAttr, memberOfAttr};
+ } else {
+ returningAttributes = new String[] {
+ groupNameAttr, posixUidAttr, posixGidAttr};
+ }
+ SEARCH_CONTROLS.setReturningAttributes(returningAttributes);
this.conf = conf;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/182fc198/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 0998b0e..74c030e 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -261,6 +261,18 @@
</property>
<property>
+ <name>hadoop.security.group.mapping.ldap.search.attr.memberof</name>
+ <value></value>
+ <description>
+ The attribute of the user object that identifies its group objects. By
+ default, Hadoop makes two LDAP queries per user if this value is empty. If
+ set, Hadoop will attempt to resolve group names from this attribute,
+ instead of making the second LDAP query to get group objects. The value
+ should be 'memberOf' for an MS AD installation.
+ </description>
+</property>
+
+<property>
<name>hadoop.security.group.mapping.ldap.search.attr.member</name>
<value>member</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/182fc198/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
index a742029..b0508f8 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -98,6 +98,12 @@ To secure the connection, the implementation supports LDAP over SSL (LDAPS). SSL
In addition, specify the path to the keystore file for SSL connection in `hadoop.security.group.mapping.ldap.ssl.keystore` and keystore password in `hadoop.security.group.mapping.ldap.ssl.keystore.password`.
Alternatively, store the keystore password in a file, and point `hadoop.security.group.mapping.ldap.ssl.keystore.password.file` to that file. For security purposes, this file should be readable only by the Unix user running the daemons.
+### Low latency group mapping resolution ###
+Typically, Hadoop resolves a user's group names by making two LDAP queries: the first query gets the user object, and the second query uses the user's Distinguished Name to find the groups.
+For some LDAP servers, such as Active Directory, the user object returned in the first query also contains the DN of the user's groups in its `memberOf` attribute, and the name of a group is its Relative Distinguished Name.
+Therefore, it is possible to infer the user's groups from the first query without sending the second one, and it may reduce group name resolution latency incurred by the second query. If it fails to get group names, it will fall back to the typical two-query scenario and send the second query to get group names.
+To enable this feature, set `hadoop.security.group.mapping.ldap.search.attr.memberof` to `memberOf`, and Hadoop will resolve group names using this attribute in the user object.
+
Composite Groups Mapping
--------
`CompositeGroupsMapping` works by enumerating a list of service providers in `hadoop.security.group.mapping.providers`.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/182fc198/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
index 93c81c7..9319016 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
@@ -19,7 +19,11 @@ package org.apache.hadoop.security;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.*;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileWriter;
@@ -31,7 +35,6 @@ import java.util.List;
import javax.naming.CommunicationException;
import javax.naming.NamingException;
import javax.naming.directory.SearchControls;
-import javax.naming.directory.SearchResult;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -47,18 +50,17 @@ import org.junit.Test;
public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
@Before
public void setupMocks() throws NamingException {
- SearchResult mockUserResult = mock(SearchResult.class);
- when(mockUserNamingEnum.nextElement()).thenReturn(mockUserResult);
- when(mockUserResult.getNameInNamespace()).thenReturn("CN=some_user,DC=test,DC=com");
+ when(getUserSearchResult().getNameInNamespace()).
+ thenReturn("CN=some_user,DC=test,DC=com");
}
@Test
public void testGetGroups() throws IOException, NamingException {
// The search functionality of the mock context is reused, so we will
// return the user NamingEnumeration first, and then the group
- when(mockContext.search(anyString(), anyString(), any(Object[].class),
+ when(getContext().search(anyString(), anyString(), any(Object[].class),
any(SearchControls.class)))
- .thenReturn(mockUserNamingEnum, mockGroupNamingEnum);
+ .thenReturn(getUserNames(), getGroupNames());
doTestGetGroups(Arrays.asList(testGroups), 2);
}
@@ -67,10 +69,10 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
public void testGetGroupsWithConnectionClosed() throws IOException, NamingException {
// The case mocks connection is closed/gc-ed, so the first search call throws CommunicationException,
// then after reconnected return the user NamingEnumeration first, and then the group
- when(mockContext.search(anyString(), anyString(), any(Object[].class),
+ when(getContext().search(anyString(), anyString(), any(Object[].class),
any(SearchControls.class)))
.thenThrow(new CommunicationException("Connection is closed"))
- .thenReturn(mockUserNamingEnum, mockGroupNamingEnum);
+ .thenReturn(getUserNames(), getGroupNames());
// Although connection is down but after reconnected it still should retrieve the result groups
doTestGetGroups(Arrays.asList(testGroups), 1 + 2); // 1 is the first failure call
@@ -79,7 +81,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
@Test
public void testGetGroupsWithLdapDown() throws IOException, NamingException {
// This mocks the case where Ldap server is down, and always throws CommunicationException
- when(mockContext.search(anyString(), anyString(), any(Object[].class),
+ when(getContext().search(anyString(), anyString(), any(Object[].class),
any(SearchControls.class)))
.thenThrow(new CommunicationException("Connection is closed"));
@@ -92,16 +94,17 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
Configuration conf = new Configuration();
// Set this, so we don't throw an exception
conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
-
- mappingSpy.setConf(conf);
+
+ LdapGroupsMapping groupsMapping = getGroupsMapping();
+ groupsMapping.setConf(conf);
// Username is arbitrary, since the spy is mocked to respond the same,
// regardless of input
- List<String> groups = mappingSpy.getGroups("some_user");
+ List<String> groups = groupsMapping.getGroups("some_user");
Assert.assertEquals(expectedGroups, groups);
// We should have searched for a user, and then two groups
- verify(mockContext, times(searchTimes)).search(anyString(),
+ verify(getContext(), times(searchTimes)).search(anyString(),
anyString(),
any(Object[].class),
any(SearchControls.class));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/182fc198/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
index c54ac4c..75e3bf1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.security;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import javax.naming.NamingEnumeration;
@@ -30,34 +29,49 @@ import javax.naming.directory.Attributes;
import javax.naming.directory.BasicAttribute;
import javax.naming.directory.BasicAttributes;
import javax.naming.directory.DirContext;
+import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
import org.junit.Before;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.mockito.Spy;
public class TestLdapGroupsMappingBase {
- protected DirContext mockContext;
+ @Mock
+ private DirContext context;
+ @Mock
+ private NamingEnumeration<SearchResult> userNames;
+ @Mock
+ private NamingEnumeration<SearchResult> groupNames;
+ @Mock
+ private SearchResult userSearchResult;
+ @Mock
+ private Attributes attributes;
+ @Spy
+ private LdapGroupsMapping groupsMapping = new LdapGroupsMapping();
- protected LdapGroupsMapping mappingSpy = spy(new LdapGroupsMapping());
- protected NamingEnumeration mockUserNamingEnum =
- mock(NamingEnumeration.class);
- protected NamingEnumeration mockGroupNamingEnum =
- mock(NamingEnumeration.class);
protected String[] testGroups = new String[] {"group1", "group2"};
@Before
public void setupMocksBase() throws NamingException {
- mockContext = mock(DirContext.class);
- doReturn(mockContext).when(mappingSpy).getDirContext();
+ MockitoAnnotations.initMocks(this);
+ DirContext ctx = getContext();
+ doReturn(ctx).when(groupsMapping).getDirContext();
+ when(ctx.search(Mockito.anyString(), Mockito.anyString(),
+ Mockito.any(Object[].class), Mockito.any(SearchControls.class))).
+ thenReturn(userNames);
// We only ever call hasMoreElements once for the user NamingEnum, so
// we can just have one return value
- when(mockUserNamingEnum.hasMoreElements()).thenReturn(true);
+ when(userNames.hasMoreElements()).thenReturn(true);
- SearchResult mockGroupResult = mock(SearchResult.class);
+ SearchResult groupSearchResult = mock(SearchResult.class);
// We're going to have to define the loop here. We want two iterations,
// to get both the groups
- when(mockGroupNamingEnum.hasMoreElements()).thenReturn(true, true, false);
- when(mockGroupNamingEnum.nextElement()).thenReturn(mockGroupResult);
+ when(groupNames.hasMoreElements()).thenReturn(true, true, false);
+ when(groupNames.nextElement()).thenReturn(groupSearchResult);
// Define the attribute for the name of the first group
Attribute group1Attr = new BasicAttribute("cn");
@@ -72,6 +86,35 @@ public class TestLdapGroupsMappingBase {
group2Attrs.put(group2Attr);
// This search result gets reused, so return group1, then group2
- when(mockGroupResult.getAttributes()).thenReturn(group1Attrs, group2Attrs);
+ when(groupSearchResult.getAttributes()).
+ thenReturn(group1Attrs, group2Attrs);
+
+ when(getUserNames().nextElement()).
+ thenReturn(getUserSearchResult());
+
+ when(getUserSearchResult().getAttributes()).thenReturn(getAttributes());
+ }
+
+ protected DirContext getContext() {
+ return context;
+ }
+ protected NamingEnumeration<SearchResult> getUserNames() {
+ return userNames;
+ }
+
+ protected NamingEnumeration<SearchResult> getGroupNames() {
+ return groupNames;
+ }
+
+ protected SearchResult getUserSearchResult() {
+ return userSearchResult;
+ }
+
+ protected Attributes getAttributes() {
+ return attributes;
+ }
+
+ protected LdapGroupsMapping getGroupsMapping() {
+ return groupsMapping;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/182fc198/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
new file mode 100644
index 0000000..e5cd2b6
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import javax.naming.NamingEnumeration;
+import javax.naming.NamingException;
+import javax.naming.directory.Attribute;
+import javax.naming.directory.SearchControls;
+import javax.naming.directory.SearchResult;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test LdapGroupsMapping with one-query lookup enabled.
+ * Mockito is used to simulate the LDAP server response.
+ */
+@SuppressWarnings("unchecked")
+public class TestLdapGroupsMappingWithOneQuery
+ extends TestLdapGroupsMappingBase {
+
+ @Before
+ public void setupMocks() throws NamingException {
+ Attribute groupDN = mock(Attribute.class);
+
+ NamingEnumeration<SearchResult> groupNames = getGroupNames();
+ doReturn(groupNames).when(groupDN).getAll();
+ String groupName1 = "CN=abc,DC=foo,DC=bar,DC=com";
+ String groupName2 = "CN=xyz,DC=foo,DC=bar,DC=com";
+ String groupName3 = "CN=sss,CN=foo,DC=bar,DC=com";
+ doReturn(groupName1).doReturn(groupName2).doReturn(groupName3).
+ when(groupNames).next();
+ when(groupNames.hasMore()).thenReturn(true).thenReturn(true).
+ thenReturn(true).thenReturn(false);
+
+ when(getAttributes().get(eq("memberOf"))).thenReturn(groupDN);
+ }
+
+ @Test
+ public void testGetGroups() throws IOException, NamingException {
+ // given a user whose ldap query returns a user object with three "memberOf"
+ // properties, return an array of strings representing its groups.
+ String[] testGroups = new String[] {"abc", "xyz", "sss"};
+ doTestGetGroups(Arrays.asList(testGroups));
+ }
+
+ private void doTestGetGroups(List<String> expectedGroups)
+ throws IOException, NamingException {
+ Configuration conf = new Configuration();
+ // Set this, so we don't throw an exception
+ conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+ // enable single-query lookup
+ conf.set(LdapGroupsMapping.MEMBEROF_ATTR_KEY, "memberOf");
+
+ LdapGroupsMapping groupsMapping = getGroupsMapping();
+ groupsMapping.setConf(conf);
+ // Username is arbitrary, since the spy is mocked to respond the same,
+ // regardless of input
+ List<String> groups = groupsMapping.getGroups("some_user");
+
+ Assert.assertEquals(expectedGroups, groups);
+
+ // We should have only made one query because single-query lookup is enabled
+ verify(getContext(), times(1)).search(anyString(),
+ anyString(),
+ any(Object[].class),
+ any(SearchControls.class));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/182fc198/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
index 247f6c4..332eed4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
@@ -36,7 +36,6 @@ import javax.naming.NamingException;
import javax.naming.directory.Attribute;
import javax.naming.directory.Attributes;
import javax.naming.directory.SearchControls;
-import javax.naming.directory.SearchResult;
import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
@@ -49,31 +48,26 @@ public class TestLdapGroupsMappingWithPosixGroup
@Before
public void setupMocks() throws NamingException {
- SearchResult mockUserResult = mock(SearchResult.class);
- when(mockUserNamingEnum.nextElement()).thenReturn(mockUserResult);
-
- Attribute mockUidNumberAttr = mock(Attribute.class);
- Attribute mockGidNumberAttr = mock(Attribute.class);
- Attribute mockUidAttr = mock(Attribute.class);
- Attributes mockAttrs = mock(Attributes.class);
-
- when(mockUidAttr.get()).thenReturn("some_user");
- when(mockUidNumberAttr.get()).thenReturn("700");
- when(mockGidNumberAttr.get()).thenReturn("600");
- when(mockAttrs.get(eq("uid"))).thenReturn(mockUidAttr);
- when(mockAttrs.get(eq("uidNumber"))).thenReturn(mockUidNumberAttr);
- when(mockAttrs.get(eq("gidNumber"))).thenReturn(mockGidNumberAttr);
-
- when(mockUserResult.getAttributes()).thenReturn(mockAttrs);
+ Attribute uidNumberAttr = mock(Attribute.class);
+ Attribute gidNumberAttr = mock(Attribute.class);
+ Attribute uidAttr = mock(Attribute.class);
+ Attributes attributes = getAttributes();
+
+ when(uidAttr.get()).thenReturn("some_user");
+ when(uidNumberAttr.get()).thenReturn("700");
+ when(gidNumberAttr.get()).thenReturn("600");
+ when(attributes.get(eq("uid"))).thenReturn(uidAttr);
+ when(attributes.get(eq("uidNumber"))).thenReturn(uidNumberAttr);
+ when(attributes.get(eq("gidNumber"))).thenReturn(gidNumberAttr);
}
@Test
public void testGetGroups() throws IOException, NamingException {
// The search functionality of the mock context is reused, so we will
// return the user NamingEnumeration first, and then the group
- when(mockContext.search(anyString(), contains("posix"),
+ when(getContext().search(anyString(), contains("posix"),
any(Object[].class), any(SearchControls.class)))
- .thenReturn(mockUserNamingEnum, mockGroupNamingEnum);
+ .thenReturn(getUserNames(), getGroupNames());
doTestGetGroups(Arrays.asList(testGroups), 2);
}
@@ -92,19 +86,20 @@ public class TestLdapGroupsMappingWithPosixGroup
conf.set(LdapGroupsMapping.POSIX_GID_ATTR_KEY, "gidNumber");
conf.set(LdapGroupsMapping.GROUP_NAME_ATTR_KEY, "cn");
- mappingSpy.setConf(conf);
+ LdapGroupsMapping groupsMapping = getGroupsMapping();
+ groupsMapping.setConf(conf);
// Username is arbitrary, since the spy is mocked to respond the same,
// regardless of input
- List<String> groups = mappingSpy.getGroups("some_user");
+ List<String> groups = groupsMapping.getGroups("some_user");
Assert.assertEquals(expectedGroups, groups);
- mappingSpy.getConf().set(LdapGroupsMapping.POSIX_UID_ATTR_KEY, "uid");
+ groupsMapping.getConf().set(LdapGroupsMapping.POSIX_UID_ATTR_KEY, "uid");
Assert.assertEquals(expectedGroups, groups);
// We should have searched for a user, and then two groups
- verify(mockContext, times(searchTimes)).search(anyString(),
+ verify(getContext(), times(searchTimes)).search(anyString(),
anyString(),
any(Object[].class),
any(SearchControls.class));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDFS-9926. MiniDFSCluster leaks
dependency Mockito via DataNodeTestUtils. (Contributed by Josh Elser)
Posted by ar...@apache.org.
HDFS-9926. MiniDFSCluster leaks dependency Mockito via DataNodeTestUtils. (Contributed by Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4d8fde8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4d8fde8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4d8fde8
Branch: refs/heads/HDFS-7240
Commit: f4d8fde8224a7154965239932a18dd563fb60f23
Parents: f2c1d91
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed May 18 09:20:57 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed May 18 09:20:57 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/TestDataTransferProtocol.java | 4 +-
.../apache/hadoop/hdfs/TestFileCorruption.java | 6 +-
.../org/apache/hadoop/hdfs/TestReplication.java | 3 +-
.../blockmanagement/TestBlockManager.java | 4 +-
.../blockmanagement/TestHeartbeatHandling.java | 14 +-
.../TestOverReplicatedBlocks.java | 8 +-
.../server/datanode/BlockReportTestBase.java | 2 +-
.../hdfs/server/datanode/DataNodeTestUtils.java | 119 +-------------
.../datanode/InternalDataNodeTestUtils.java | 157 +++++++++++++++++++
.../datanode/TestDataNodeHotSwapVolumes.java | 2 +-
.../datanode/TestDataNodeMetricsLogger.java | 2 +-
.../datanode/TestDataNodeReconfiguration.java | 2 +-
...TestDnRespectsBlockReportSplitThreshold.java | 6 +-
.../server/datanode/TestFsDatasetCache.java | 2 +-
.../datanode/TestIncrementalBlockReports.java | 2 +-
.../hdfs/server/datanode/TestStorageReport.java | 2 +-
.../server/datanode/TestTriggerBlockReport.java | 2 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 12 +-
.../hdfs/server/namenode/TestDeleteRace.java | 4 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 5 +-
.../namenode/ha/TestPipelinesFailover.java | 6 +-
21 files changed, 206 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
index e0ee3e7..aeff16d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
@@ -59,7 +59,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ReadOpChecksumIn
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.DataChecksum;
@@ -209,7 +209,7 @@ public class TestDataTransferProtocol {
try {
cluster.waitActive();
String poolId = cluster.getNamesystem().getBlockPoolId();
- datanode = DataNodeTestUtils.getDNRegistrationForBP(
+ datanode = InternalDataNodeTestUtils.getDNRegistrationForBP(
cluster.getDataNodes().get(0), poolId);
dnAddr = NetUtils.createSocketAddr(datanode.getXferAddr());
FileSystem fileSys = cluster.getFileSystem();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
index 011baa1..2437e38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCorruption.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -154,8 +154,8 @@ public class TestFileCorruption {
DataNode dataNode = datanodes.get(2);
// report corrupted block by the third datanode
- DatanodeRegistration dnR =
- DataNodeTestUtils.getDNRegistrationForBP(dataNode, blk.getBlockPoolId());
+ DatanodeRegistration dnR = InternalDataNodeTestUtils.
+ getDNRegistrationForBP(dataNode, blk.getBlockPoolId());
FSNamesystem ns = cluster.getNamesystem();
ns.writeLock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index e0e0456..3e23db3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils.MaterializedReplica;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -571,7 +572,7 @@ public class TestReplication {
NameNode nn = cluster.getNameNode();
DataNode dn = cluster.getDataNodes().get(0);
DatanodeProtocolClientSideTranslatorPB spy =
- DataNodeTestUtils.spyOnBposToNN(dn, nn);
+ InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG);
Mockito.doAnswer(delayer).when(spy).blockReceivedAndDeleted(
Mockito.<DatanodeRegistration>anyObject(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 3ee0810..394fae9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -74,7 +74,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -950,7 +950,7 @@ public class TestBlockManager {
final FSNamesystem namesystem = cluster.getNamesystem();
final String poolId = namesystem.getBlockPoolId();
final DatanodeRegistration nodeReg =
- DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().
+ InternalDataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().
get(0), poolId);
final DatanodeDescriptor dd = NameNodeAdapter.getDatanode(namesystem,
nodeReg);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
index b77c413..ab607ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
@@ -66,7 +66,8 @@ public class TestHeartbeatHandling {
).getDatanodeManager().getHeartbeatManager();
final String poolId = namesystem.getBlockPoolId();
final DatanodeRegistration nodeReg =
- DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
+ InternalDataNodeTestUtils.
+ getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
final DatanodeDescriptor dd = NameNodeAdapter.getDatanode(namesystem, nodeReg);
final String storageID = DatanodeStorage.generateUuid();
dd.updateStorage(new DatanodeStorage(storageID));
@@ -149,15 +150,18 @@ public class TestHeartbeatHandling {
).getDatanodeManager().getHeartbeatManager();
final String poolId = namesystem.getBlockPoolId();
final DatanodeRegistration nodeReg1 =
- DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
+ InternalDataNodeTestUtils.
+ getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
final DatanodeDescriptor dd1 = NameNodeAdapter.getDatanode(namesystem, nodeReg1);
dd1.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid()));
final DatanodeRegistration nodeReg2 =
- DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(1), poolId);
+ InternalDataNodeTestUtils.
+ getDNRegistrationForBP(cluster.getDataNodes().get(1), poolId);
final DatanodeDescriptor dd2 = NameNodeAdapter.getDatanode(namesystem, nodeReg2);
dd2.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid()));
final DatanodeRegistration nodeReg3 =
- DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(2), poolId);
+ InternalDataNodeTestUtils.
+ getDNRegistrationForBP(cluster.getDataNodes().get(2), poolId);
final DatanodeDescriptor dd3 = NameNodeAdapter.getDatanode(namesystem, nodeReg3);
dd3.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
index e33c7a3..d5f2fb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -88,7 +88,7 @@ public class TestOverReplicatedBlocks {
String blockPoolId = cluster.getNamesystem().getBlockPoolId();
final DatanodeID corruptDataNode =
- DataNodeTestUtils.getDNRegistrationForBP(
+ InternalDataNodeTestUtils.getDNRegistrationForBP(
cluster.getDataNodes().get(2), blockPoolId);
final FSNamesystem namesystem = cluster.getNamesystem();
@@ -157,8 +157,8 @@ public class TestOverReplicatedBlocks {
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 300);
cluster.startDataNodes(conf, 1, true, null, null, null);
DataNode lastDN = cluster.getDataNodes().get(3);
- DatanodeRegistration dnReg = DataNodeTestUtils.getDNRegistrationForBP(
- lastDN, namesystem.getBlockPoolId());
+ DatanodeRegistration dnReg = InternalDataNodeTestUtils.
+ getDNRegistrationForBP(lastDN, namesystem.getBlockPoolId());
String lastDNid = dnReg.getDatanodeUuid();
final Path fileName = new Path("/foo2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
index 4279f96..53b9263 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/BlockReportTestBase.java
@@ -614,7 +614,7 @@ public abstract class BlockReportTestBase {
// from this node.
DataNode dn = cluster.getDataNodes().get(0);
DatanodeProtocolClientSideTranslatorPB spy =
- DataNodeTestUtils.spyOnBposToNN(dn, nn);
+ InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
Mockito.doAnswer(delayer)
.when(spy).blockReport(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
index 1d47192..e2755f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
@@ -19,44 +19,21 @@
package org.apache.hadoop.hdfs.server.datanode;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.File;
import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
import org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol;
-import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat;
-import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
-import org.apache.hadoop.hdfs.server.protocol.StorageReport;
-import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
-import org.junit.Assert;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import com.google.common.base.Preconditions;
/**
* Utility class for accessing package-private DataNode information during tests.
- *
+ * Must not contain usage of classes that are not explicitly listed as
+ * dependencies to {@link MiniDFSCluster}.
*/
public class DataNodeTestUtils {
private static final String DIR_FAILURE_SUFFIX = ".origin";
@@ -101,41 +78,6 @@ public class DataNodeTestUtils {
bpos.triggerBlockReportForTests();
}
}
-
- /**
- * Insert a Mockito spy object between the given DataNode and
- * the given NameNode. This can be used to delay or wait for
- * RPC calls on the datanode->NN path.
- */
- public static DatanodeProtocolClientSideTranslatorPB spyOnBposToNN(
- DataNode dn, NameNode nn) {
- String bpid = nn.getNamesystem().getBlockPoolId();
-
- BPOfferService bpos = null;
- for (BPOfferService thisBpos : dn.getAllBpOs()) {
- if (thisBpos.getBlockPoolId().equals(bpid)) {
- bpos = thisBpos;
- break;
- }
- }
- Preconditions.checkArgument(bpos != null,
- "No such bpid: %s", bpid);
-
- BPServiceActor bpsa = null;
- for (BPServiceActor thisBpsa : bpos.getBPServiceActors()) {
- if (thisBpsa.getNNSocketAddress().equals(nn.getServiceRpcAddress())) {
- bpsa = thisBpsa;
- break;
- }
- }
- Preconditions.checkArgument(bpsa != null,
- "No service actor to NN at %s", nn.getServiceRpcAddress());
-
- DatanodeProtocolClientSideTranslatorPB origNN = bpsa.getNameNodeProxy();
- DatanodeProtocolClientSideTranslatorPB spy = Mockito.spy(origNN);
- bpsa.setNameNode(spy);
- return spy;
- }
public static InterDatanodeProtocol createInterDatanodeProtocolProxy(
DataNode dn, DatanodeID datanodeid, final Configuration conf,
@@ -233,61 +175,4 @@ public class DataNodeTestUtils {
dn.getDirectoryScanner().reconcile();
}
}
-
- /**
- * Starts an instance of DataNode with NN mocked. Called should ensure to
- * shutdown the DN
- *
- * @throws IOException
- */
- public static DataNode startDNWithMockNN(Configuration conf,
- final InetSocketAddress nnSocketAddr, final String dnDataDir)
- throws IOException {
-
- FileSystem.setDefaultUri(conf, "hdfs://" + nnSocketAddr.getHostName() + ":"
- + nnSocketAddr.getPort());
- ArrayList<StorageLocation> locations = new ArrayList<StorageLocation>();
- File dataDir = new File(dnDataDir);
- FileUtil.fullyDelete(dataDir);
- dataDir.mkdirs();
- StorageLocation location = StorageLocation.parse(dataDir.getPath());
- locations.add(location);
-
- final DatanodeProtocolClientSideTranslatorPB namenode =
- mock(DatanodeProtocolClientSideTranslatorPB.class);
-
- Mockito.doAnswer(new Answer<DatanodeRegistration>() {
- @Override
- public DatanodeRegistration answer(InvocationOnMock invocation)
- throws Throwable {
- return (DatanodeRegistration) invocation.getArguments()[0];
- }
- }).when(namenode).registerDatanode(Mockito.any(DatanodeRegistration.class));
-
- when(namenode.versionRequest()).thenReturn(
- new NamespaceInfo(1, TEST_CLUSTER_ID, TEST_POOL_ID, 1L));
-
- when(
- namenode.sendHeartbeat(Mockito.any(DatanodeRegistration.class),
- Mockito.any(StorageReport[].class), Mockito.anyLong(),
- Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt(),
- Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class),
- Mockito.anyBoolean())).thenReturn(
- new HeartbeatResponse(new DatanodeCommand[0], new NNHAStatusHeartbeat(
- HAServiceState.ACTIVE, 1), null, ThreadLocalRandom.current()
- .nextLong() | 1L));
-
- DataNode dn = new DataNode(conf, locations, null) {
- @Override
- DatanodeProtocolClientSideTranslatorPB connectToNN(
- InetSocketAddress nnAddr) throws IOException {
- Assert.assertEquals(nnSocketAddr, nnAddr);
- return namenode;
- }
- };
- // Trigger a heartbeat so that it acknowledges the NN as active.
- dn.getAllBpOs().get(0).triggerHeartbeatForTests();
-
- return dn;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
new file mode 100644
index 0000000..2f400e2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
+import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat;
+import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
+import org.junit.Assert;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * An internal-facing only collection of test utilities for the DataNode. This
+ * is to ensure that test-scope dependencies aren't inadvertently leaked
+ * to clients, e.g. Mockito.
+ */
+public class InternalDataNodeTestUtils {
+
+ public final static String TEST_CLUSTER_ID = "testClusterID";
+ public final static String TEST_POOL_ID = "BP-TEST";
+
+ public static DatanodeRegistration
+ getDNRegistrationForBP(DataNode dn, String bpid) throws IOException {
+ return dn.getDNRegistrationForBP(bpid);
+ }
+
+ /**
+ * Insert a Mockito spy object between the given DataNode and
+ * the given NameNode. This can be used to delay or wait for
+ * RPC calls on the datanode->NN path.
+ */
+ public static DatanodeProtocolClientSideTranslatorPB spyOnBposToNN(
+ DataNode dn, NameNode nn) {
+ String bpid = nn.getNamesystem().getBlockPoolId();
+
+ BPOfferService bpos = null;
+ for (BPOfferService thisBpos : dn.getAllBpOs()) {
+ if (thisBpos.getBlockPoolId().equals(bpid)) {
+ bpos = thisBpos;
+ break;
+ }
+ }
+ Preconditions.checkArgument(bpos != null,
+ "No such bpid: %s", bpid);
+
+ BPServiceActor bpsa = null;
+ for (BPServiceActor thisBpsa : bpos.getBPServiceActors()) {
+ if (thisBpsa.getNNSocketAddress().equals(nn.getServiceRpcAddress())) {
+ bpsa = thisBpsa;
+ break;
+ }
+ }
+ Preconditions.checkArgument(bpsa != null,
+ "No service actor to NN at %s", nn.getServiceRpcAddress());
+
+ DatanodeProtocolClientSideTranslatorPB origNN = bpsa.getNameNodeProxy();
+ DatanodeProtocolClientSideTranslatorPB spy = Mockito.spy(origNN);
+ bpsa.setNameNode(spy);
+ return spy;
+ }
+
+ /**
+ * Starts an instance of DataNode with NN mocked. Called should ensure to
+ * shutdown the DN
+ *
+ * @throws IOException
+ */
+ public static DataNode startDNWithMockNN(Configuration conf,
+ final InetSocketAddress nnSocketAddr, final String dnDataDir)
+ throws IOException {
+
+ FileSystem.setDefaultUri(conf, "hdfs://" + nnSocketAddr.getHostName() + ":"
+ + nnSocketAddr.getPort());
+ ArrayList<StorageLocation> locations = new ArrayList<StorageLocation>();
+ File dataDir = new File(dnDataDir);
+ FileUtil.fullyDelete(dataDir);
+ dataDir.mkdirs();
+ StorageLocation location = StorageLocation.parse(dataDir.getPath());
+ locations.add(location);
+
+ final DatanodeProtocolClientSideTranslatorPB namenode =
+ mock(DatanodeProtocolClientSideTranslatorPB.class);
+
+ Mockito.doAnswer(new Answer<DatanodeRegistration>() {
+ @Override
+ public DatanodeRegistration answer(InvocationOnMock invocation)
+ throws Throwable {
+ return (DatanodeRegistration) invocation.getArguments()[0];
+ }
+ }).when(namenode).registerDatanode(Mockito.any(DatanodeRegistration.class));
+
+ when(namenode.versionRequest()).thenReturn(
+ new NamespaceInfo(1, TEST_CLUSTER_ID, TEST_POOL_ID,
+ 1L));
+
+ when(
+ namenode.sendHeartbeat(Mockito.any(DatanodeRegistration.class),
+ Mockito.any(StorageReport[].class), Mockito.anyLong(),
+ Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt(),
+ Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class),
+ Mockito.anyBoolean())).thenReturn(
+ new HeartbeatResponse(new DatanodeCommand[0], new NNHAStatusHeartbeat(
+ HAServiceState.ACTIVE, 1), null, ThreadLocalRandom.current()
+ .nextLong() | 1L));
+
+ DataNode dn = new DataNode(conf, locations, null) {
+ @Override
+ DatanodeProtocolClientSideTranslatorPB connectToNN(
+ InetSocketAddress nnAddr) throws IOException {
+ Assert.assertEquals(nnSocketAddr, nnAddr);
+ return namenode;
+ }
+ };
+ // Trigger a heartbeat so that it acknowledges the NN as active.
+ dn.getAllBpOs().get(0).triggerHeartbeatForTests();
+
+ return dn;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 8315392..659806b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -822,7 +822,7 @@ public class TestDataNodeHotSwapVolumes {
final DataNode dn = cluster.getDataNodes().get(0);
DatanodeProtocolClientSideTranslatorPB spy =
- DataNodeTestUtils.spyOnBposToNN(dn, cluster.getNameNode());
+ InternalDataNodeTestUtils.spyOnBposToNN(dn, cluster.getNameNode());
// Remove a data dir from datanode
File dataDirToKeep = new File(cluster.getDataDirectory(), "data1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetricsLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetricsLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetricsLogger.java
index 1177a45..32fda37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetricsLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetricsLogger.java
@@ -89,7 +89,7 @@ public class TestDataNodeMetricsLogger {
conf.setInt(DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_KEY,
enableMetricsLogging ? 1 : 0); // If enabled, log early and log often
- dn = DataNodeTestUtils.startDNWithMockNN(conf, NN_ADDR, DATA_DIR);
+ dn = InternalDataNodeTestUtils.startDNWithMockNN(conf, NN_ADDR, DATA_DIR);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java
index 249b5c6..1dfd3c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java
@@ -102,7 +102,7 @@ public class TestDataNodeReconfiguration {
DataNode[] result = new DataNode[numDateNode];
for (int i = 0; i < numDateNode; i++) {
- result[i] = DataNodeTestUtils.startDNWithMockNN(conf, NN_ADDR, DATA_DIR);
+ result[i] = InternalDataNodeTestUtils.startDNWithMockNN(conf, NN_ADDR, DATA_DIR);
}
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDnRespectsBlockReportSplitThreshold.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDnRespectsBlockReportSplitThreshold.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDnRespectsBlockReportSplitThreshold.java
index badd593..bf0e3c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDnRespectsBlockReportSplitThreshold.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDnRespectsBlockReportSplitThreshold.java
@@ -122,7 +122,7 @@ public class TestDnRespectsBlockReportSplitThreshold {
// Insert a spy object for the NN RPC.
DatanodeProtocolClientSideTranslatorPB nnSpy =
- DataNodeTestUtils.spyOnBposToNN(dn, nn);
+ InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
// Trigger a block report so there is an interaction with the spy
// object.
@@ -154,7 +154,7 @@ public class TestDnRespectsBlockReportSplitThreshold {
// Insert a spy object for the NN RPC.
DatanodeProtocolClientSideTranslatorPB nnSpy =
- DataNodeTestUtils.spyOnBposToNN(dn, nn);
+ InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
// Trigger a block report so there is an interaction with the spy
// object.
@@ -186,7 +186,7 @@ public class TestDnRespectsBlockReportSplitThreshold {
// Insert a spy object for the NN RPC.
DatanodeProtocolClientSideTranslatorPB nnSpy =
- DataNodeTestUtils.spyOnBposToNN(dn, nn);
+ InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
// Trigger a block report so there is an interaction with the spy
// object.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
index 10d974c..6557055 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
@@ -140,7 +140,7 @@ public class TestFsDatasetCache {
dn = cluster.getDataNodes().get(0);
fsd = dn.getFSDataset();
- spyNN = DataNodeTestUtils.spyOnBposToNN(dn, nn);
+ spyNN = InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
index 2288974..03553fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
@@ -106,7 +106,7 @@ public class TestIncrementalBlockReports {
* @return spy object that can be used for Mockito verification.
*/
DatanodeProtocolClientSideTranslatorPB spyOnDnCallsToNn() {
- return DataNodeTestUtils.spyOnBposToNN(singletonDn, singletonNn);
+ return InternalDataNodeTestUtils.spyOnBposToNN(singletonDn, singletonNn);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java
index a6032c1..d8418d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java
@@ -92,7 +92,7 @@ public class TestStorageReport {
// Insert a spy object for the NN RPC.
DatanodeProtocolClientSideTranslatorPB nnSpy =
- DataNodeTestUtils.spyOnBposToNN(dn, nn);
+ InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
// Trigger a heartbeat so there is an interaction with the spy
// object.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java
index 83ebeeb..f7c9911 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestTriggerBlockReport.java
@@ -63,7 +63,7 @@ public final class TestTriggerBlockReport {
cluster.waitActive();
FileSystem fs = cluster.getFileSystem();
DatanodeProtocolClientSideTranslatorPB spy =
- DataNodeTestUtils.spyOnBposToNN(
+ InternalDataNodeTestUtils.spyOnBposToNN(
cluster.getDataNodes().get(0), cluster.getNameNode());
DFSTestUtil.createFile(fs, new Path("/abc"), 16, (short) 1, 1L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index 04f0b91..341a149 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
@@ -86,9 +86,9 @@ public class TestDeadDatanode {
String poolId = cluster.getNamesystem().getBlockPoolId();
// wait for datanode to be marked live
DataNode dn = cluster.getDataNodes().get(0);
- DatanodeRegistration reg =
- DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
-
+ DatanodeRegistration reg = InternalDataNodeTestUtils.
+ getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
+
DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), true, 20000);
// Shutdown and wait for datanode to be marked dead
@@ -147,8 +147,8 @@ public class TestDeadDatanode {
String poolId = cluster.getNamesystem().getBlockPoolId();
// wait for datanode to be marked live
DataNode dn = cluster.getDataNodes().get(0);
- DatanodeRegistration reg = DataNodeTestUtils.getDNRegistrationForBP(cluster
- .getDataNodes().get(0), poolId);
+ DatanodeRegistration reg = InternalDataNodeTestUtils.
+ getDNRegistrationForBP(cluster.getDataNodes().get(0), poolId);
// Get the updated datanode descriptor
BlockManager bm = cluster.getNamesystem().getBlockManager();
DatanodeManager dm = bm.getDatanodeManager();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
index 15f697a..133a18e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.Node;
@@ -299,7 +299,7 @@ public class TestDeleteRace {
DataNode primaryDN = cluster.getDataNode(expectedPrimary.getIpcPort());
DatanodeProtocolClientSideTranslatorPB nnSpy = dnMap.get(primaryDN);
if (nnSpy == null) {
- nnSpy = DataNodeTestUtils.spyOnBposToNN(primaryDN, nn);
+ nnSpy = InternalDataNodeTestUtils.spyOnBposToNN(primaryDN, nn);
dnMap.put(primaryDN, nnSpy);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
index 3b3c35f..15357b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
@@ -539,8 +540,8 @@ public class TestDNFencing {
DataNode dn = cluster.getDataNodes().get(0);
DatanodeProtocolClientSideTranslatorPB spy =
- DataNodeTestUtils.spyOnBposToNN(dn, nn2);
-
+ InternalDataNodeTestUtils.spyOnBposToNN(dn, nn2);
+
Mockito.doAnswer(delayer)
.when(spy).blockReport(
Mockito.<DatanodeRegistration>anyObject(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d8fde8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
index 9ece121..4fc5029 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.IOUtils;
@@ -358,8 +358,8 @@ public class TestPipelinesFailover {
// active.
DataNode primaryDN = cluster.getDataNode(expectedPrimary.getIpcPort());
DatanodeProtocolClientSideTranslatorPB nnSpy =
- DataNodeTestUtils.spyOnBposToNN(primaryDN, nn0);
-
+ InternalDataNodeTestUtils.spyOnBposToNN(primaryDN, nn0);
+
// Delay the commitBlockSynchronization call
DelayAnswer delayer = new DelayAnswer(LOG);
Mockito.doAnswer(delayer).when(nnSpy).commitBlockSynchronization(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-12767. Update apache
httpclient version to 4.5.2; httpcore to 4.4.4. Artem Aliev via stevel.
Posted by ar...@apache.org.
HADOOP-12767. Update apache httpclient version to 4.5.2; httpcore to 4.4.4. Artem Aliev via stevel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22ff9e68
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22ff9e68
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22ff9e68
Branch: refs/heads/HDFS-7240
Commit: 22ff9e68d1d9f27d62455c15fb1242538551aca9
Parents: c7159f0
Author: Steve Loughran <st...@apache.org>
Authored: Thu May 19 14:44:44 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu May 19 14:48:18 2016 +0100
----------------------------------------------------------------------
.../web/DelegationTokenAuthenticationFilter.java | 7 +++++--
.../security/token/delegation/web/ServletUtils.java | 7 +++++--
hadoop-project/pom.xml | 4 ++--
.../hadoop/yarn/server/webproxy/WebAppProxyServlet.java | 11 +++++++----
4 files changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ff9e68/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index af66ee5..fb6817e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -216,8 +216,11 @@ public class DelegationTokenAuthenticationFilter
@VisibleForTesting
static String getDoAs(HttpServletRequest request) {
- List<NameValuePair> list = URLEncodedUtils.parse(request.getQueryString(),
- UTF8_CHARSET);
+ String queryString = request.getQueryString();
+ if (queryString == null) {
+ return null;
+ }
+ List<NameValuePair> list = URLEncodedUtils.parse(queryString, UTF8_CHARSET);
if (list != null) {
for (NameValuePair nv : list) {
if (DelegationTokenAuthenticatedURL.DO_AS.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ff9e68/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/ServletUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/ServletUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/ServletUtils.java
index 16137ac..078dfa4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/ServletUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/ServletUtils.java
@@ -45,8 +45,11 @@ class ServletUtils {
*/
public static String getParameter(HttpServletRequest request, String name)
throws IOException {
- List<NameValuePair> list = URLEncodedUtils.parse(request.getQueryString(),
- UTF8_CHARSET);
+ String queryString = request.getQueryString();
+ if (queryString == null) {
+ return null;
+ }
+ List<NameValuePair> list = URLEncodedUtils.parse(queryString, UTF8_CHARSET);
if (list != null) {
for (NameValuePair nv : list) {
if (name.equals(nv.getName())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ff9e68/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 46ecb23..3a2f9d9 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -462,12 +462,12 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.2.5</version>
+ <version>4.5.2</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>4.2.5</version>
+ <version>4.4.4</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22ff9e68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
index 9d64667..0b621aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
@@ -423,11 +423,14 @@ public class WebAppProxyServlet extends HttpServlet {
// Append the user-provided path and query parameter to the original
// tracking url.
- List<NameValuePair> queryPairs =
- URLEncodedUtils.parse(req.getQueryString(), null);
UriBuilder builder = UriBuilder.fromUri(trackingUri);
- for (NameValuePair pair : queryPairs) {
- builder.queryParam(pair.getName(), pair.getValue());
+ String queryString = req.getQueryString();
+ if (queryString != null) {
+ List<NameValuePair> queryPairs =
+ URLEncodedUtils.parse(queryString, null);
+ for (NameValuePair pair : queryPairs) {
+ builder.queryParam(pair.getName(), pair.getValue());
+ }
}
URI toFetch = builder.path(rest).build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDFS-10303.
DataStreamer#ResponseProcessor calculates packet ack latency incorrectly.
Contributed by Surendra Singh Lilhore.
Posted by ar...@apache.org.
HDFS-10303. DataStreamer#ResponseProcessor calculates packet ack latency incorrectly. Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a5819da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a5819da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a5819da
Branch: refs/heads/HDFS-7240
Commit: 4a5819dae2b0ca8f8b6d94ef464882d079d86593
Parents: 08ea07f
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue May 17 08:53:31 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue May 17 08:53:31 2016 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DataStreamer.java | 28 +++++--
.../org/apache/hadoop/hdfs/TestDataStream.java | 85 ++++++++++++++++++++
2 files changed, 105 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a5819da/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 4d5f16c..1b7306a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -32,10 +32,12 @@ import java.net.Socket;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -395,6 +397,7 @@ class DataStreamer extends Daemon {
private volatile boolean appendChunk = false;
// both dataQueue and ackQueue are protected by dataQueue lock
protected final LinkedList<DFSPacket> dataQueue = new LinkedList<>();
+ private final Map<Long, Long> packetSendTime = new HashMap<>();
private final LinkedList<DFSPacket> ackQueue = new LinkedList<>();
private final AtomicReference<CachingStrategy> cachingStrategy;
private final ByteArrayManager byteArrayManager;
@@ -644,6 +647,7 @@ class DataStreamer extends Daemon {
scope = null;
dataQueue.removeFirst();
ackQueue.addLast(one);
+ packetSendTime.put(one.getSeqno(), Time.monotonicNow());
dataQueue.notifyAll();
}
}
@@ -957,15 +961,21 @@ class DataStreamer extends Daemon {
// process responses from datanodes.
try {
// read an ack from the pipeline
- long begin = Time.monotonicNow();
ack.readFields(blockReplyStream);
- long duration = Time.monotonicNow() - begin;
- if (duration > dfsclientSlowLogThresholdMs
- && ack.getSeqno() != DFSPacket.HEART_BEAT_SEQNO) {
- LOG.warn("Slow ReadProcessor read fields took " + duration
- + "ms (threshold=" + dfsclientSlowLogThresholdMs + "ms); ack: "
- + ack + ", targets: " + Arrays.asList(targets));
- } else {
+ if (ack.getSeqno() != DFSPacket.HEART_BEAT_SEQNO) {
+ Long begin = packetSendTime.get(ack.getSeqno());
+ if (begin != null) {
+ long duration = Time.monotonicNow() - begin;
+ if (duration > dfsclientSlowLogThresholdMs) {
+ LOG.info("Slow ReadProcessor read fields for block " + block
+ + " took " + duration + "ms (threshold="
+ + dfsclientSlowLogThresholdMs + "ms); ack: " + ack
+ + ", targets: " + Arrays.asList(targets));
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
LOG.debug("DFSClient {}", ack);
}
@@ -1047,6 +1057,7 @@ class DataStreamer extends Daemon {
lastAckedSeqno = seqno;
pipelineRecoveryCount = 0;
ackQueue.removeFirst();
+ packetSendTime.remove(seqno);
dataQueue.notifyAll();
one.releaseBuffer(byteArrayManager);
@@ -1105,6 +1116,7 @@ class DataStreamer extends Daemon {
synchronized (dataQueue) {
dataQueue.addAll(0, ackQueue);
ackQueue.clear();
+ packetSendTime.clear();
}
// If we had to recover the pipeline five times in a row for the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a5819da/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataStream.java
new file mode 100644
index 0000000..3351b68
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataStream.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestDataStream {
+ static MiniDFSCluster cluster;
+ static int PACKET_SIZE = 1024;
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setInt(HdfsClientConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY,
+ PACKET_SIZE);
+ conf.setInt(HdfsClientConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY,
+ 10000);
+ conf.setInt(HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 60000);
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ }
+
+ @Test(timeout = 60000)
+ public void testDfsClient() throws IOException, InterruptedException {
+ LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(LogFactory
+ .getLog(DataStreamer.class));
+ byte[] toWrite = new byte[PACKET_SIZE];
+ new Random(1).nextBytes(toWrite);
+ final Path path = new Path("/file1");
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ FSDataOutputStream out = null;
+ out = dfs.create(path, false);
+
+ out.write(toWrite);
+ out.write(toWrite);
+ out.hflush();
+
+ //Wait to cross slow IO warning threshold
+ Thread.sleep(15 * 1000);
+
+ out.write(toWrite);
+ out.write(toWrite);
+ out.hflush();
+
+ //Wait for capturing logs in busy cluster
+ Thread.sleep(5 * 1000);
+
+ out.close();
+ logs.stopCapturing();
+ GenericTestUtils.assertDoesNotMatch(logs.getOutput(),
+ "Slow ReadProcessor read fields for block");
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ cluster.shutdown();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: Revert "HDFS-10417. Improve error
message from checkBlockLocalPathAccess. Contributed by Tianyin Xu."
Posted by ar...@apache.org.
Revert "HDFS-10417. Improve error message from checkBlockLocalPathAccess. Contributed by Tianyin Xu."
This reverts commit 0942954e8ab6bef636c2648d0d5a0227a50f799c.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1356cbe9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1356cbe9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1356cbe9
Branch: refs/heads/HDFS-7240
Commit: 1356cbe9941c1692c6614201e63187916a3eebb0
Parents: 0942954
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue May 17 15:20:03 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue May 17 15:20:03 2016 -0500
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/datanode/DataNode.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1356cbe9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 410472e..288fc3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1717,8 +1717,7 @@ public class DataNode extends ReconfigurableBase
throw new AccessControlException(
"Can't continue with getBlockLocalPathInfo() "
+ "authorization. The user " + currentUser
- + " is not configured in "
- + DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY);
+ + " is not allowed to call getBlockLocalPathInfo");
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HADOOP-13177. Native tests fail on
OS X,
because DYLD_LIBRARY_PATH is not defined to include libhadoop.dylib (Chris
Nauroth via aw)
Posted by ar...@apache.org.
HADOOP-13177. Native tests fail on OS X, because DYLD_LIBRARY_PATH is not defined to include libhadoop.dylib (Chris Nauroth via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7159f02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7159f02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7159f02
Branch: refs/heads/HDFS-7240
Commit: c7159f0240be0c555edf01d9a580987b0997125d
Parents: d4274c6
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu May 19 06:25:22 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu May 19 06:25:22 2016 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7159f02/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index f5fb8af..46ecb23 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1159,6 +1159,7 @@
<!-- HADOOP_HOME required for tests on Windows to find winutils -->
<HADOOP_HOME>${hadoop.common.build.dir}</HADOOP_HOME>
<LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib</LD_LIBRARY_PATH>
+ <DYLD_LIBRARY_PATH>${env.DYLD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib</DYLD_LIBRARY_PATH>
<MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
</environmentVariables>
<systemPropertyVariables>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-5107. TestContainerMetrics
fails. (aajisaka)
Posted by ar...@apache.org.
YARN-5107. TestContainerMetrics fails. (aajisaka)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4274c64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4274c64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4274c64
Branch: refs/heads/HDFS-7240
Commit: d4274c64bc77584e2a75107c1f141319ddc4f8ab
Parents: 1597630
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu May 19 19:53:59 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu May 19 19:53:59 2016 +0900
----------------------------------------------------------------------
.../containermanager/monitor/TestContainerMetrics.java | 11 -----------
1 file changed, 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4274c64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
index 60f6604..6966489 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainerMetrics.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsRecord;
-import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.impl.MetricsCollectorImpl;
import org.apache.hadoop.metrics2.impl.MetricsRecords;
import org.apache.hadoop.metrics2.impl.MetricsSystemImpl;
@@ -37,9 +36,6 @@ import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
public class TestContainerMetrics {
@@ -48,10 +44,6 @@ public class TestContainerMetrics {
public void testContainerMetricsFlow() throws InterruptedException {
final String ERR = "Error in number of records";
- // Create a dummy MetricsSystem
- MetricsSystem system = mock(MetricsSystem.class);
- doReturn(this).when(system).register(anyString(), anyString(), any());
-
MetricsCollectorImpl collector = new MetricsCollectorImpl();
ContainerId containerId = mock(ContainerId.class);
ContainerMetrics metrics = ContainerMetrics.forContainer(containerId,
@@ -89,9 +81,6 @@ public class TestContainerMetrics {
public void testContainerMetricsLimit() throws InterruptedException {
final String ERR = "Error in number of records";
- MetricsSystem system = mock(MetricsSystem.class);
- doReturn(this).when(system).register(anyString(), anyString(), any());
-
MetricsCollectorImpl collector = new MetricsCollectorImpl();
ContainerId containerId = mock(ContainerId.class);
ContainerMetrics metrics = ContainerMetrics.forContainer(containerId,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-10381,
DataStreamer DataNode exclusion log message should be warning. (John
Zhuge via Yongjun Zhang)
Posted by ar...@apache.org.
HDFS-10381, DataStreamer DataNode exclusion log message should be warning. (John Zhuge via Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/16c07cc6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/16c07cc6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/16c07cc6
Branch: refs/heads/HDFS-7240
Commit: 16c07cc68a3e0a06f57b7f4c7207cc8e5dce211f
Parents: 03788d3
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Tue May 17 15:45:15 2016 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Tue May 17 15:57:33 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DataStreamer.java | 4 ++--
.../main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16c07cc6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 1b7306a..ee673ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -1518,12 +1518,12 @@ class DataStreamer extends Daemon {
success = createBlockOutputStream(nodes, storageTypes, 0L, false);
if (!success) {
- LOG.info("Abandoning " + block);
+ LOG.warn("Abandoning " + block);
dfsClient.namenode.abandonBlock(block, stat.getFileId(), src,
dfsClient.clientName);
block = null;
final DatanodeInfo badNode = nodes[errorState.getBadNodeIndex()];
- LOG.info("Excluding datanode " + badNode);
+ LOG.warn("Excluding datanode " + badNode);
excludedNodes.put(badNode, badNode);
}
} while (!success && --count >= 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16c07cc6/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
index 8f4efcc..89ab6a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
@@ -107,7 +107,7 @@ public class StripedDataStreamer extends DataStreamer {
if (!success) {
block = null;
final DatanodeInfo badNode = nodes[getErrorState().getBadNodeIndex()];
- LOG.info("Excluding datanode " + badNode);
+ LOG.warn("Excluding datanode " + badNode);
excludedNodes.put(badNode, badNode);
throw new IOException("Unable to create new block." + this);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-5110. Fix
OpportunisticContainerAllocator to insert complete HostAddress in issued
ContainerTokenIds. (Konstantinos Karanasos via asuresh)
Posted by ar...@apache.org.
YARN-5110. Fix OpportunisticContainerAllocator to insert complete HostAddress in issued ContainerTokenIds. (Konstantinos Karanasos via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15976306
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15976306
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15976306
Branch: refs/heads/HDFS-7240
Commit: 1597630681c784a3d59f5605b87e96197b8139d7
Parents: 010e6ac
Author: Arun Suresh <as...@apache.org>
Authored: Wed May 18 18:46:00 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Wed May 18 18:46:00 2016 -0700
----------------------------------------------------------------------
.../api/records/QueuedContainersStatus.java | 2 +-
.../queuing/QueuingContainerManagerImpl.java | 1 +
.../nodemanager/scheduler/LocalScheduler.java | 6 +-
.../OpportunisticContainerAllocator.java | 17 +++---
.../scheduler/TestLocalScheduler.java | 9 ++-
.../distributed/NodeQueueLoadMonitor.java | 14 +++--
.../hadoop/yarn/server/MiniYARNCluster.java | 62 +++++++++++++++++++-
7 files changed, 91 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java
index a7f0ece..fb567d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/QueuedContainersStatus.java
@@ -41,5 +41,5 @@ public abstract class QueuedContainersStatus {
public abstract int getWaitQueueLength();
- public abstract void setWaitQueueLength(int queueWaitTime);
+ public abstract void setWaitQueueLength(int waitQueueLength);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
index 4f9d5a3..707051f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/queuing/QueuingContainerManagerImpl.java
@@ -200,6 +200,7 @@ public class QueuingContainerManagerImpl extends ContainerManagerImpl {
.getContainerTokenIdentifier().getContainerID();
this.context.getQueuingContext().getQueuedContainers().remove(containerId);
try {
+ LOG.info("Starting container [" + containerId + "]");
super.startContainerInternal(
allocatedContainerInfo.getContainerTokenIdentifier(),
allocatedContainerInfo.getStartRequest());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/LocalScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/LocalScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/LocalScheduler.java
index 42c1dcd..fca814b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/LocalScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/LocalScheduler.java
@@ -337,8 +337,10 @@ public final class LocalScheduler extends AbstractRequestInterceptor {
@Override
public DistSchedAllocateResponse allocateForDistributedScheduling
(AllocateRequest request) throws YarnException, IOException {
- LOG.info("Forwarding allocate request to the" +
- "Distributed Scheduler Service on YARN RM");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Forwarding allocate request to the" +
+ "Distributed Scheduler Service on YARN RM");
+ }
// Partition requests into GUARANTEED and OPPORTUNISTIC reqs
PartitionedResourceRequests partitionedAsks = partitionAskList(request
.getAskList());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java
index 03ba61d..e33c389 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/OpportunisticContainerAllocator.java
@@ -84,14 +84,14 @@ public class OpportunisticContainerAllocator {
Map<String, NodeId> allNodes, String userName) throws YarnException {
Map<Resource, List<Container>> containers = new HashMap<>();
Set<String> nodesAllocated = new HashSet<>();
- int numAsks = resourceAsks.size();
for (ResourceRequest anyAsk : resourceAsks) {
allocateOpportunisticContainers(appParams, idCounter, blacklist, appAttId,
allNodes, userName, containers, nodesAllocated, anyAsk);
- }
- if (numAsks > 0) {
- LOG.info("Opportunistic allocation requested for: " + numAsks
- + " containers; allocated = " + containers.size());
+ LOG.info("Opportunistic allocation requested for ["
+ + "priority=" + anyAsk.getPriority()
+ + ", num_containers=" + anyAsk.getNumContainers()
+ + ", capability=" + anyAsk.getCapability() + "]"
+ + " allocated = " + containers.get(anyAsk.getCapability()).size());
}
return containers;
}
@@ -129,8 +129,9 @@ public class OpportunisticContainerAllocator {
}
cList.add(container);
numAllocated++;
- LOG.info("Allocated " + numAllocated + " opportunistic containers.");
+ LOG.info("Allocated [" + container.getId() + "] as opportunistic.");
}
+ LOG.info("Allocated " + numAllocated + " opportunistic containers.");
}
private Container buildContainer(DistSchedulerParams appParams,
@@ -146,8 +147,8 @@ public class OpportunisticContainerAllocator {
long currTime = System.currentTimeMillis();
ContainerTokenIdentifier containerTokenIdentifier =
new ContainerTokenIdentifier(
- cId, nodeId.getHost(), userName, capability,
- currTime + appParams.containerTokenExpiryInterval,
+ cId, nodeId.getHost() + ":" + nodeId.getPort(), userName,
+ capability, currTime + appParams.containerTokenExpiryInterval,
context.getContainerTokenSecretManager().getCurrentKey().getKeyId(),
nodeStatusUpdater.getRMIdentifier(), rr.getPriority(), currTime,
null, CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/TestLocalScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/TestLocalScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/TestLocalScheduler.java
index efc682a..e987e79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/TestLocalScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/scheduler/TestLocalScheduler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.DistributedSchedulerProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.server.nodemanager.security
.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security
.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Test;
@@ -196,9 +198,14 @@ public class TestLocalScheduler {
}
private Map<NodeId, List<ContainerId>> mapAllocs(AllocateResponse
- allocateResponse) {
+ allocateResponse) throws Exception {
Map<NodeId, List<ContainerId>> allocs = new HashMap<>();
for (Container c : allocateResponse.getAllocatedContainers()) {
+ ContainerTokenIdentifier cTokId = BuilderUtils
+ .newContainerTokenIdentifier(c.getContainerToken());
+ Assert.assertEquals(
+ c.getNodeId().getHost() + ":" + c.getNodeId().getPort(),
+ cTokId.getNmHostAddress());
List<ContainerId> cIds = allocs.get(c.getNodeId());
if (cIds == null) {
cIds = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
index 21f4f6e..017a256 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
@@ -195,11 +195,11 @@ public class NodeQueueLoadMonitor implements ClusterMonitor {
new ClusterNode(rmNode.getNodeID())
.setQueueWaitTime(estimatedQueueWaitTime)
.setQueueLength(waitQueueLength));
- LOG.info("Inserting ClusterNode [" + rmNode.getNodeID() + "]" +
+ LOG.info("Inserting ClusterNode [" + rmNode.getNodeID() + "] " +
"with queue wait time [" + estimatedQueueWaitTime + "] and " +
"wait queue length [" + waitQueueLength + "]");
} else {
- LOG.warn("IGNORING ClusterNode [" + rmNode.getNodeID() + "]" +
+ LOG.warn("IGNORING ClusterNode [" + rmNode.getNodeID() + "] " +
"with queue wait time [" + estimatedQueueWaitTime + "] and " +
"wait queue length [" + waitQueueLength + "]");
}
@@ -210,12 +210,14 @@ public class NodeQueueLoadMonitor implements ClusterMonitor {
.setQueueWaitTime(estimatedQueueWaitTime)
.setQueueLength(waitQueueLength)
.updateTimestamp();
- LOG.info("Updating ClusterNode [" + rmNode.getNodeID() + "]" +
- "with queue wait time [" + estimatedQueueWaitTime + "] and " +
- "wait queue length [" + waitQueueLength + "]");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating ClusterNode [" + rmNode.getNodeID() + "] " +
+ "with queue wait time [" + estimatedQueueWaitTime + "] and " +
+ "wait queue length [" + waitQueueLength + "]");
+ }
} else {
this.clusterNodes.remove(rmNode.getNodeID());
- LOG.info("Deleting ClusterNode [" + rmNode.getNodeID() + "]" +
+ LOG.info("Deleting ClusterNode [" + rmNode.getNodeID() + "] " +
"with queue wait time [" + currentNode.queueWaitTime + "] and " +
"wait queue length [" + currentNode.queueLength + "]");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15976306/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 2372ea2..de4e22d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -75,6 +75,9 @@ import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.AMRMProxyService;
import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor;
import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.RequestInterceptor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.queuing.QueuingContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService;
@@ -713,8 +716,14 @@ public class MiniYARNCluster extends CompositeService {
ContainerExecutor exec, DeletionService del,
NodeStatusUpdater nodeStatusUpdater, ApplicationACLsManager aclsManager,
LocalDirsHandlerService dirsHandler) {
- return new CustomContainerManagerImpl(context, exec, del,
- nodeStatusUpdater, metrics, dirsHandler);
+ if (getConfig().getBoolean(YarnConfiguration.NM_CONTAINER_QUEUING_ENABLED,
+ YarnConfiguration.NM_CONTAINER_QUEUING_ENABLED_DEFAULT)) {
+ return new CustomQueueingContainerManagerImpl(context, exec, del,
+ nodeStatusUpdater, metrics, dirsHandler);
+ } else {
+ return new CustomContainerManagerImpl(context, exec, del,
+ nodeStatusUpdater, metrics, dirsHandler);
+ }
}
}
@@ -846,6 +855,55 @@ public class MiniYARNCluster extends CompositeService {
}
}
+ private class CustomQueueingContainerManagerImpl extends
+ QueuingContainerManagerImpl {
+
+ public CustomQueueingContainerManagerImpl(Context context,
+ ContainerExecutor exec, DeletionService del, NodeStatusUpdater
+ nodeStatusUpdater, NodeManagerMetrics metrics,
+ LocalDirsHandlerService dirsHandler) {
+ super(context, exec, del, nodeStatusUpdater, metrics, dirsHandler);
+ }
+
+ @Override
+ protected ContainersMonitor createContainersMonitor(ContainerExecutor
+ exec) {
+ return new ContainersMonitorImpl(exec, dispatcher, this.context) {
+
+ @Override
+ public void increaseContainersAllocation(ProcessTreeInfo pti) { }
+
+ @Override
+ public void decreaseContainersAllocation(ProcessTreeInfo pti) { }
+
+ @Override
+ public boolean hasResourcesAvailable(
+ ContainersMonitorImpl.ProcessTreeInfo pti) {
+ return true;
+ }
+ };
+ }
+
+ @Override
+ protected void createAMRMProxyService(Configuration conf) {
+ this.amrmProxyEnabled =
+ conf.getBoolean(YarnConfiguration.AMRM_PROXY_ENABLED,
+ YarnConfiguration.DEFAULT_AMRM_PROXY_ENABLED);
+
+ if (this.amrmProxyEnabled) {
+ LOG.info("CustomAMRMProxyService is enabled. "
+ + "All the AM->RM requests will be intercepted by the proxy");
+ AMRMProxyService amrmProxyService =
+ useRpc ? new AMRMProxyService(getContext(), dispatcher)
+ : new ShortCircuitedAMRMProxy(getContext(), dispatcher);
+ this.setAMRMProxyService(amrmProxyService);
+ addService(this.getAMRMProxyService());
+ } else {
+ LOG.info("CustomAMRMProxyService is disabled");
+ }
+ }
+ }
+
private class ShortCircuitedAMRMProxy extends AMRMProxyService {
public ShortCircuitedAMRMProxy(Context context,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDFS-10383. Safely close resources
in DFSTestUtil. Contributed by Mingliang Liu.
Posted by ar...@apache.org.
HDFS-10383. Safely close resources in DFSTestUtil. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd99f5fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd99f5fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd99f5fc
Branch: refs/heads/HDFS-7240
Commit: dd99f5fc7934c6501995c94b630a7c7e97e880d5
Parents: 16c07cc
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue May 17 16:12:00 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue May 17 16:12:00 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 278 +++++++++----------
1 file changed, 126 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd99f5fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index eda9910..f80cd78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
@@ -147,6 +148,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStat
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.hdfs.tools.JMXGet;
+import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
@@ -314,9 +316,10 @@ public class DFSTestUtil {
}
public static byte[] readFileAsBytes(FileSystem fs, Path fileName) throws IOException {
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- IOUtils.copyBytes(fs.open(fileName), os, 1024, true);
- return os.toByteArray();
+ try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
+ IOUtils.copyBytes(fs.open(fileName), os, 1024);
+ return os.toByteArray();
+ }
}
/** create nFiles with random names and directory hierarchies
@@ -346,17 +349,10 @@ public class DFSTestUtil {
public static byte[] readFileBuffer(FileSystem fs, Path fileName)
throws IOException {
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- try {
- FSDataInputStream in = fs.open(fileName);
- try {
- IOUtils.copyBytes(in, os, 1024, true);
- return os.toByteArray();
- } finally {
- in.close();
- }
- } finally {
- os.close();
+ try (ByteArrayOutputStream os = new ByteArrayOutputStream();
+ FSDataInputStream in = fs.open(fileName)) {
+ IOUtils.copyBytes(in, os, 1024, true);
+ return os.toByteArray();
}
}
@@ -384,51 +380,40 @@ public class DFSTestUtil {
boolean isLazyPersist, int bufferLen, long fileLen, long blockSize,
short replFactor, long seed, boolean flush,
InetSocketAddress[] favoredNodes) throws IOException {
- assert bufferLen > 0;
- if (!fs.mkdirs(fileName.getParent())) {
+ assert bufferLen > 0;
+ if (!fs.mkdirs(fileName.getParent())) {
throw new IOException("Mkdirs failed to create " +
- fileName.getParent().toString());
- }
- FSDataOutputStream out = null;
- EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
- createFlags.add(OVERWRITE);
- if (isLazyPersist) {
- createFlags.add(LAZY_PERSIST);
- }
- try {
- if (favoredNodes == null) {
- out = fs.create(
- fileName,
- FsPermission.getFileDefault(),
- createFlags,
- fs.getConf().getInt(
- CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
- replFactor, blockSize, null);
- } else {
- out = ((DistributedFileSystem) fs).create(fileName,
- FsPermission.getDefault(), true, bufferLen, replFactor, blockSize,
- null, favoredNodes);
+ fileName.getParent().toString());
}
+ EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
+ createFlags.add(OVERWRITE);
+ if (isLazyPersist) {
+ createFlags.add(LAZY_PERSIST);
+ }
+ try (FSDataOutputStream out = (favoredNodes == null) ?
+ fs.create(fileName, FsPermission.getFileDefault(), createFlags,
+ fs.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, 4096), replFactor,
+ blockSize, null)
+ :
+ ((DistributedFileSystem) fs).create(fileName, FsPermission.getDefault(),
+ true, bufferLen, replFactor, blockSize, null, favoredNodes)
+ ) {
if (fileLen > 0) {
byte[] toWrite = new byte[bufferLen];
Random rb = new Random(seed);
long bytesToWrite = fileLen;
- while (bytesToWrite>0) {
+ while (bytesToWrite > 0) {
rb.nextBytes(toWrite);
int bytesToWriteNext = (bufferLen < bytesToWrite) ? bufferLen
- : (int) bytesToWrite;
+ : (int) bytesToWrite;
- out.write(toWrite, 0, bytesToWriteNext);
- bytesToWrite -= bytesToWriteNext;
+ out.write(toWrite, 0, bytesToWriteNext);
+ bytesToWrite -= bytesToWriteNext;
}
if (flush) {
out.hsync();
}
}
- } finally {
- if (out != null) {
- out.close();
- }
}
}
@@ -445,20 +430,18 @@ public class DFSTestUtil {
for (int idx = 0; idx < nFiles; idx++) {
Path fPath = new Path(root, files[idx].getName());
- FSDataInputStream in = fs.open(fPath);
- byte[] toRead = new byte[files[idx].getSize()];
- byte[] toCompare = new byte[files[idx].getSize()];
- Random rb = new Random(files[idx].getSeed());
- rb.nextBytes(toCompare);
- in.readFully(0, toRead);
- in.close();
- for (int i = 0; i < toRead.length; i++) {
- if (toRead[i] != toCompare[i]) {
- return false;
+ try (FSDataInputStream in = fs.open(fPath)) {
+ byte[] toRead = new byte[files[idx].getSize()];
+ byte[] toCompare = new byte[files[idx].getSize()];
+ Random rb = new Random(files[idx].getSeed());
+ rb.nextBytes(toCompare);
+ in.readFully(0, toRead);
+ for (int i = 0; i < toRead.length; i++) {
+ if (toRead[i] != toCompare[i]) {
+ return false;
+ }
}
}
- toRead = null;
- toCompare = null;
}
return true;
@@ -492,16 +475,13 @@ public class DFSTestUtil {
*/
public static boolean allBlockReplicasCorrupt(MiniDFSCluster cluster,
Path file, int blockNo) throws IOException {
- DFSClient client = new DFSClient(new InetSocketAddress("localhost",
- cluster.getNameNodePort()), cluster.getConfiguration(0));
- LocatedBlocks blocks;
- try {
- blocks = client.getNamenode().getBlockLocations(
+ try (DFSClient client = new DFSClient(new InetSocketAddress("localhost",
+ cluster.getNameNodePort()), cluster.getConfiguration(0))) {
+ LocatedBlocks blocks;
+ blocks = client.getNamenode().getBlockLocations(
file.toString(), 0, Long.MAX_VALUE);
- } finally {
- client.close();
+ return blocks.get(blockNo).isCorrupt();
}
- return blocks.get(blockNo).isCorrupt();
}
/*
@@ -781,12 +761,9 @@ public class DFSTestUtil {
}
public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
- HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
- try {
+ try (HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path)) {
in.readByte();
return in.getCurrentBlock();
- } finally {
- in.close();
}
}
@@ -797,8 +774,9 @@ public class DFSTestUtil {
public static List<LocatedBlock> getAllBlocks(FileSystem fs, Path path)
throws IOException {
- HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
- return in.getAllBlocks();
+ try (HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path)) {
+ return in.getAllBlocks();
+ }
}
public static Token<BlockTokenIdentifier> getBlockToken(
@@ -807,17 +785,21 @@ public class DFSTestUtil {
}
public static String readFile(File f) throws IOException {
- StringBuilder b = new StringBuilder();
- BufferedReader in = new BufferedReader(new FileReader(f));
- for(int c; (c = in.read()) != -1; b.append((char)c));
- in.close();
- return b.toString();
+ try (BufferedReader in = new BufferedReader(new FileReader(f))) {
+ StringBuilder b = new StringBuilder();
+ int c;
+ while ((c = in.read()) != -1) {
+ b.append((char) c);
+ }
+ return b.toString();
+ }
}
public static byte[] readFileAsBytes(File f) throws IOException {
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- IOUtils.copyBytes(new FileInputStream(f), os, 1024, true);
- return os.toByteArray();
+ try (ByteArrayOutputStream os = new ByteArrayOutputStream()) {
+ IOUtils.copyBytes(new FileInputStream(f), os, 1024);
+ return os.toByteArray();
+ }
}
/* Write the given bytes to the given file */
@@ -826,9 +808,10 @@ public class DFSTestUtil {
if (fs.exists(p)) {
fs.delete(p, true);
}
- InputStream is = new ByteArrayInputStream(bytes);
- FSDataOutputStream os = fs.create(p);
- IOUtils.copyBytes(is, os, bytes.length, true);
+ try (InputStream is = new ByteArrayInputStream(bytes);
+ FSDataOutputStream os = fs.create(p)) {
+ IOUtils.copyBytes(is, os, bytes.length);
+ }
}
/* Write the given string to the given file */
@@ -841,9 +824,10 @@ public class DFSTestUtil {
public static void appendFile(FileSystem fs, Path p, String s)
throws IOException {
assert fs.exists(p);
- InputStream is = new ByteArrayInputStream(s.getBytes());
- FSDataOutputStream os = fs.append(p);
- IOUtils.copyBytes(is, os, s.length(), true);
+ try (InputStream is = new ByteArrayInputStream(s.getBytes());
+ FSDataOutputStream os = fs.append(p)) {
+ IOUtils.copyBytes(is, os, s.length());
+ }
}
/**
@@ -860,9 +844,9 @@ public class DFSTestUtil {
byte[] toAppend = new byte[length];
Random random = new Random();
random.nextBytes(toAppend);
- FSDataOutputStream out = fs.append(p);
- out.write(toAppend);
- out.close();
+ try (FSDataOutputStream out = fs.append(p)) {
+ out.write(toAppend);
+ }
}
/**
@@ -980,35 +964,32 @@ public class DFSTestUtil {
*/
public static byte[] loadFile(String filename) throws IOException {
File file = new File(filename);
- DataInputStream in = new DataInputStream(new FileInputStream(file));
- byte[] content = new byte[(int)file.length()];
- try {
+ try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {
+ byte[] content = new byte[(int) file.length()];
in.readFully(content);
- } finally {
- IOUtils.cleanup(LOG, in);
+ return content;
}
- return content;
}
/** For {@link TestTransferRbw} */
public static BlockOpResponseProto transferRbw(final ExtendedBlock b,
final DFSClient dfsClient, final DatanodeInfo... datanodes) throws IOException {
assertEquals(2, datanodes.length);
- final Socket s = DataStreamer.createSocketForPipeline(datanodes[0],
- datanodes.length, dfsClient);
final long writeTimeout = dfsClient.getDatanodeWriteTimeout(datanodes.length);
- final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
- NetUtils.getOutputStream(s, writeTimeout),
- DFSUtilClient.getSmallBufferSize(dfsClient.getConfiguration())));
- final DataInputStream in = new DataInputStream(NetUtils.getInputStream(s));
-
- // send the request
- new Sender(out).transferBlock(b, new Token<BlockTokenIdentifier>(),
- dfsClient.clientName, new DatanodeInfo[]{datanodes[1]},
- new StorageType[]{StorageType.DEFAULT});
- out.flush();
-
- return BlockOpResponseProto.parseDelimitedFrom(in);
+ try (Socket s = DataStreamer.createSocketForPipeline(datanodes[0],
+ datanodes.length, dfsClient);
+ DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
+ NetUtils.getOutputStream(s, writeTimeout),
+ DFSUtilClient.getSmallBufferSize(dfsClient.getConfiguration())));
+ DataInputStream in = new DataInputStream(NetUtils.getInputStream(s))) {
+ // send the request
+ new Sender(out).transferBlock(b, new Token<BlockTokenIdentifier>(),
+ dfsClient.clientName, new DatanodeInfo[]{datanodes[1]},
+ new StorageType[]{StorageType.DEFAULT});
+ out.flush();
+
+ return BlockOpResponseProto.parseDelimitedFrom(in);
+ }
}
public static void setFederatedConfiguration(MiniDFSCluster cluster,
@@ -1556,13 +1537,12 @@ public class DFSTestUtil {
*/
public static void verifyFilesEqual(FileSystem fs, Path p1, Path p2, int len)
throws IOException {
- final FSDataInputStream in1 = fs.open(p1);
- final FSDataInputStream in2 = fs.open(p2);
- for (int i = 0; i < len; i++) {
- assertEquals("Mismatch at byte " + i, in1.read(), in2.read());
+ try (FSDataInputStream in1 = fs.open(p1);
+ FSDataInputStream in2 = fs.open(p2)) {
+ for (int i = 0; i < len; i++) {
+ assertEquals("Mismatch at byte " + i, in1.read(), in2.read());
+ }
}
- in1.close();
- in2.close();
}
/**
@@ -1575,20 +1555,15 @@ public class DFSTestUtil {
* @throws IOException
*/
public static void verifyFilesNotEqual(FileSystem fs, Path p1, Path p2,
- int len)
- throws IOException {
- final FSDataInputStream in1 = fs.open(p1);
- final FSDataInputStream in2 = fs.open(p2);
- try {
+ int len) throws IOException {
+ try (FSDataInputStream in1 = fs.open(p1);
+ FSDataInputStream in2 = fs.open(p2)) {
for (int i = 0; i < len; i++) {
if (in1.read() != in2.read()) {
return;
}
}
fail("files are equal, but should not be");
- } finally {
- in1.close();
- in2.close();
}
}
@@ -1699,13 +1674,13 @@ public class DFSTestUtil {
int ret = 0;
try {
ByteArrayOutputStream bs = new ByteArrayOutputStream(1024);
- PrintStream out = new PrintStream(bs);
- System.setOut(out);
- System.setErr(out);
- ret = tool.run(cmds);
- System.out.flush();
- System.err.flush();
- out.close();
+ try (PrintStream out = new PrintStream(bs)) {
+ System.setOut(out);
+ System.setErr(out);
+ ret = tool.run(cmds);
+ System.out.flush();
+ System.err.flush();
+ }
output = bs.toString();
} finally {
System.setOut(origOut);
@@ -1798,9 +1773,9 @@ public class DFSTestUtil {
ExtendedBlock blk, int dnIndex, int lenDelta) throws IOException {
File blockFile = cluster.getBlockFile(dnIndex, blk);
if (blockFile != null && blockFile.exists()) {
- RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
- raFile.setLength(raFile.length()+lenDelta);
- raFile.close();
+ try (RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw")) {
+ raFile.setLength(raFile.length() + lenDelta);
+ }
return true;
}
LOG.info("failed to change length of block " + blk);
@@ -1925,27 +1900,26 @@ public class DFSTestUtil {
}
}
- FSDataOutputStream out = null;
- try {
- out = dfs.create(file, (short) 1); // create an empty file
-
- FSNamesystem ns = cluster.getNamesystem();
- FSDirectory fsdir = ns.getFSDirectory();
- INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
-
- ExtendedBlock previous = null;
- for (int i = 0; i < numBlocks; i++) {
- Block newBlock = addBlockToFile(true, cluster.getDataNodes(), dfs, ns,
- file.toString(), fileNode, dfs.getClient().getClientName(),
- previous, numStripesPerBlk, 0);
- previous = new ExtendedBlock(ns.getBlockPoolId(), newBlock);
- }
-
- dfs.getClient().namenode.complete(file.toString(),
- dfs.getClient().getClientName(), previous, fileNode.getId());
- } finally {
- IOUtils.cleanup(null, out);
+ cluster.getNameNodeRpc()
+ .create(file.toString(), new FsPermission((short)0755),
+ dfs.getClient().getClientName(),
+ new EnumSetWritable<>(EnumSet.of(CreateFlag.CREATE)),
+ false, (short)1, 128*1024*1024L, null);
+
+ FSNamesystem ns = cluster.getNamesystem();
+ FSDirectory fsdir = ns.getFSDirectory();
+ INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
+
+ ExtendedBlock previous = null;
+ for (int i = 0; i < numBlocks; i++) {
+ Block newBlock = addBlockToFile(true, cluster.getDataNodes(), dfs, ns,
+ file.toString(), fileNode, dfs.getClient().getClientName(),
+ previous, numStripesPerBlk, 0);
+ previous = new ExtendedBlock(ns.getBlockPoolId(), newBlock);
}
+
+ dfs.getClient().namenode.complete(file.toString(),
+ dfs.getClient().getClientName(), previous, fileNode.getId());
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HADOOP-11858. [JDK8] Set minimum
version of Hadoop 3 to JDK 8. Contributed by Robert Kanter.
Posted by ar...@apache.org.
HADOOP-11858. [JDK8] Set minimum version of Hadoop 3 to JDK 8. Contributed by Robert Kanter.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b55642b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b55642b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b55642b
Branch: refs/heads/HDFS-7240
Commit: 4b55642b9d836691592405805c181d12c2ed7e50
Parents: a9a8297
Author: Andrew Wang <wa...@apache.org>
Authored: Mon May 16 15:45:39 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon May 16 15:45:39 2016 -0700
----------------------------------------------------------------------
BUILDING.txt | 8 ++++----
hadoop-project/pom.xml | 4 ++--
pom.xml | 2 +-
3 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b55642b/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index c7a91da..9d297f7 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -4,7 +4,7 @@ Build instructions for Hadoop
Requirements:
* Unix System
-* JDK 1.7+
+* JDK 1.8+
* Maven 3.0 or later
* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.5.0
@@ -56,12 +56,12 @@ Known issues:
----------------------------------------------------------------------------------
Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
-* Oracle JDK 1.7 (preferred)
+* Oracle JDK 1.8 (preferred)
$ sudo apt-get purge openjdk*
$ sudo apt-get install software-properties-common
$ sudo add-apt-repository ppa:webupd8team/java
$ sudo apt-get update
- $ sudo apt-get install oracle-java7-installer
+ $ sudo apt-get install oracle-java8-installer
* Maven
$ sudo apt-get -y install maven
* Native libraries
@@ -306,7 +306,7 @@ Building on Windows
Requirements:
* Windows System
-* JDK 1.7+
+* JDK 1.8+
* Maven 3.0 or later
* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.5.0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b55642b/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7994ded..f5fb8af 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -86,7 +86,7 @@
<tomcat.version>6.0.44</tomcat.version>
<!-- define the Java language version used by the compiler -->
- <javac.version>1.7</javac.version>
+ <javac.version>1.8</javac.version>
<!-- The java version enforced by the maven enforcer -->
<!-- more complex patterns can be used here, such as
@@ -97,7 +97,7 @@
<enforced.maven.version>[3.0.2,)</enforced.maven.version>
<!-- Plugin versions and config -->
- <maven-surefire-plugin.argLine>-Xmx2048m -XX:MaxPermSize=768m -XX:+HeapDumpOnOutOfMemoryError</maven-surefire-plugin.argLine>
+ <maven-surefire-plugin.argLine>-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError</maven-surefire-plugin.argLine>
<maven-surefire-plugin.version>2.17</maven-surefire-plugin.version>
<maven-surefire-report-plugin.version>${maven-surefire-plugin.version}</maven-surefire-report-plugin.version>
<maven-failsafe-plugin.version>${maven-surefire-plugin.version}</maven-failsafe-plugin.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b55642b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d2fc6fd..9756133 100644
--- a/pom.xml
+++ b/pom.xml
@@ -144,7 +144,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<version>[3.0.2,)</version>
</requireMavenVersion>
<requireJavaVersion>
- <version>[1.7,)</version>
+ <version>[1.8,)</version>
</requireJavaVersion>
</rules>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org