You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2014/11/17 07:30:03 UTC
[01/45] hadoop git commit: YARN-1964. Create Docker analog of the
LinuxContainerExecutor in YARN
Repository: hadoop
Updated Branches:
refs/heads/HDFS-EC 95b3ebaa1 -> 041e97d79
YARN-1964. Create Docker analog of the LinuxContainerExecutor in YARN
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53f64ee5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53f64ee5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53f64ee5
Branch: refs/heads/HDFS-EC
Commit: 53f64ee516d03f6ec87b41d77c214aa2fe4fa0ed
Parents: 46f6f9d
Author: Ravi Prakash <ra...@apache.org>
Authored: Tue Nov 11 21:28:11 2014 -0800
Committer: Ravi Prakash <ra...@apache.org>
Committed: Tue Nov 11 21:28:11 2014 -0800
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 1 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 14 +-
.../src/main/resources/yarn-default.xml | 12 +-
.../server/nodemanager/ContainerExecutor.java | 31 +
.../nodemanager/DockerContainerExecutor.java | 794 +++++++++++++++++++
.../launcher/ContainerLaunch.java | 36 +-
.../TestDockerContainerExecutor.java | 213 +++++
.../TestDockerContainerExecutorWithMocks.java | 259 ++++++
.../launcher/TestContainerLaunch.java | 10 +-
.../src/site/apt/DockerContainerExecutor.apt.vm | 200 +++++
11 files changed, 1533 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 6a61a83..4a2c2f8 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -124,6 +124,7 @@
<item name="YARN Commands" href="hadoop-yarn/hadoop-yarn-site/YarnCommands.html"/>
<item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
<item name="NodeManager Restart" href="hadoop-yarn/hadoop-yarn-site/NodeManagerRestart.html"/>
+ <item name="DockerContainerExecutor" href="hadoop-yarn/hadoop-yarn-site/DockerContainerExecutor.html"/>
</menu>
<menu name="YARN REST APIs" inherit="top">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b40d9b7..c3bdf3d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -89,6 +89,9 @@ Release 2.6.0 - 2014-11-15
NEW FEATURES
+ YARN-1964. Create Docker analog of the LinuxContainerExecutor in YARN. (Abin
+ Shahab via raviprak)
+
YARN-2131. Add a way to format the RMStateStore. (Robert Kanter via kasha)
YARN-1367. Changed NM to not kill containers on NM resync if RM work-preserving
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b459ee3..9b57a42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -891,7 +891,19 @@ public class YarnConfiguration extends Configuration {
/** The arguments to pass to the health check script.*/
public static final String NM_HEALTH_CHECK_SCRIPT_OPTS =
NM_PREFIX + "health-checker.script.opts";
-
+
+ /** The Docker image name(For DockerContainerExecutor).*/
+ public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME =
+ NM_PREFIX + "docker-container-executor.image-name";
+
+ /** The name of the docker executor (For DockerContainerExecutor).*/
+ public static final String NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME =
+ NM_PREFIX + "docker-container-executor.exec-name";
+
+ /** The default docker executor (For DockerContainerExecutor).*/
+ public static final String NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME =
+ "/usr/bin/docker";
+
/** The path to the Linux container executor.*/
public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
NM_PREFIX + "linux-container-executor.path";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1e7d544..af4a5eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -23,7 +23,7 @@
<!-- there. If yarn-site.xml does not already exist, create it. -->
<configuration>
-
+
<!-- IPC Configs -->
<property>
<description>Factory to create client IPC classes.</description>
@@ -1146,6 +1146,16 @@
<value>${hadoop.tmp.dir}/yarn-nm-recovery</value>
</property>
+ <!--Docker configuration-->
+
+ <property>
+ <name>yarn.nodemanager.docker-container-executor.exec-name</name>
+ <value>/usr/bin/docker</value>
+ <description>
+ Name or path to the Docker client.
+ </description>
+ </property>
+
<!--Map Reduce configuration-->
<property>
<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 4ce1a75..8133413 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.yarn.server.nodemanager;
import java.io.File;
import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -211,6 +214,34 @@ public abstract class ContainerExecutor implements Configurable {
}
}
+ public void writeLaunchEnv(OutputStream out, Map<String, String> environment, Map<Path, List<String>> resources, List<String> command) throws IOException{
+ ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder.create();
+ if (environment != null) {
+ for (Map.Entry<String,String> env : environment.entrySet()) {
+ sb.env(env.getKey().toString(), env.getValue().toString());
+ }
+ }
+ if (resources != null) {
+ for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
+ for (String linkName : entry.getValue()) {
+ sb.symlink(entry.getKey(), new Path(linkName));
+ }
+ }
+ }
+
+ sb.command(command);
+
+ PrintStream pout = null;
+ try {
+ pout = new PrintStream(out);
+ sb.write(pout);
+ } finally {
+ if (out != null) {
+ out.close();
+ }
+ }
+ }
+
public enum ExitCode {
FORCE_KILLED(137),
TERMINATED(143),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
new file mode 100644
index 0000000..d8dd890
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -0,0 +1,794 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.net.InetSocketAddress;
+
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+
+/**
+ * This executor will launch a docker container and run the task inside the container.
+ */
+public class DockerContainerExecutor extends ContainerExecutor {
+
+ private static final Log LOG = LogFactory
+ .getLog(DockerContainerExecutor.class);
+ public static final String DOCKER_CONTAINER_EXECUTOR_SCRIPT = "docker_container_executor";
+ public static final String DOCKER_CONTAINER_EXECUTOR_SESSION_SCRIPT = "docker_container_executor_session";
+
+ // This validates that the image is a proper docker image and would not crash docker.
+ public static final String DOCKER_IMAGE_PATTERN = "^(([\\w\\.-]+)(:\\d+)*\\/)?[\\w\\.:-]+$";
+
+
+ private final FileContext lfs;
+ private final Pattern dockerImagePattern;
+
+ public DockerContainerExecutor() {
+ try {
+ this.lfs = FileContext.getLocalFSFileContext();
+ this.dockerImagePattern = Pattern.compile(DOCKER_IMAGE_PATTERN);
+ } catch (UnsupportedFileSystemException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void copyFile(Path src, Path dst, String owner) throws IOException {
+ lfs.util().copy(src, dst);
+ }
+
+ @Override
+ public void init() throws IOException {
+ String auth = getConf().get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
+ if (auth != null && !auth.equals("simple")) {
+ throw new IllegalStateException("DockerContainerExecutor only works with simple authentication mode");
+ }
+ String dockerExecutor = getConf().get(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME,
+ YarnConfiguration.NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME);
+ if (!new File(dockerExecutor).exists()) {
+ throw new IllegalStateException("Invalid docker exec path: " + dockerExecutor);
+ }
+ }
+
+ @Override
+ public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
+ InetSocketAddress nmAddr, String user, String appId, String locId,
+ LocalDirsHandlerService dirsHandler)
+ throws IOException, InterruptedException {
+
+ List<String> localDirs = dirsHandler.getLocalDirs();
+ List<String> logDirs = dirsHandler.getLogDirs();
+
+ ContainerLocalizer localizer =
+ new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
+ RecordFactoryProvider.getRecordFactory(getConf()));
+
+ createUserLocalDirs(localDirs, user);
+ createUserCacheDirs(localDirs, user);
+ createAppDirs(localDirs, user, appId);
+ createAppLogDirs(appId, logDirs, user);
+
+ // randomly choose the local directory
+ Path appStorageDir = getWorkingDir(localDirs, user, appId);
+
+ String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
+ Path tokenDst = new Path(appStorageDir, tokenFn);
+ copyFile(nmPrivateContainerTokensPath, tokenDst, user);
+ LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
+ lfs.setWorkingDirectory(appStorageDir);
+ LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
+ // TODO: DO it over RPC for maintaining similarity?
+ localizer.runLocalization(nmAddr);
+ }
+
+
+ @Override
+ public int launchContainer(Container container,
+ Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath,
+ String userName, String appId, Path containerWorkDir,
+ List<String> localDirs, List<String> logDirs) throws IOException {
+ String containerImageName = container.getLaunchContext().getEnvironment()
+ .get(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("containerImageName from launchContext: " + containerImageName);
+ }
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(containerImageName), "Container image must not be null");
+ containerImageName = containerImageName.replaceAll("['\"]", "");
+
+ Preconditions.checkArgument(saneDockerImage(containerImageName), "Image: " + containerImageName + " is not a proper docker image");
+ String dockerExecutor = getConf().get(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME,
+ YarnConfiguration.NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME);
+
+ FsPermission dirPerm = new FsPermission(APPDIR_PERM);
+ ContainerId containerId = container.getContainerId();
+
+ // create container dirs on all disks
+ String containerIdStr = ConverterUtils.toString(containerId);
+ String appIdStr =
+ ConverterUtils.toString(
+ containerId.getApplicationAttemptId().
+ getApplicationId());
+ for (String sLocalDir : localDirs) {
+ Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
+ Path userdir = new Path(usersdir, userName);
+ Path appCacheDir = new Path(userdir, ContainerLocalizer.APPCACHE);
+ Path appDir = new Path(appCacheDir, appIdStr);
+ Path containerDir = new Path(appDir, containerIdStr);
+ createDir(containerDir, dirPerm, true, userName);
+ }
+
+ // Create the container log-dirs on all disks
+ createContainerLogDirs(appIdStr, containerIdStr, logDirs, userName);
+
+ Path tmpDir = new Path(containerWorkDir,
+ YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
+ createDir(tmpDir, dirPerm, false, userName);
+
+ // copy launch script to work dir
+ Path launchDst =
+ new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
+ lfs.util().copy(nmPrivateContainerScriptPath, launchDst);
+
+ // copy container tokens to work dir
+ Path tokenDst =
+ new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE);
+ lfs.util().copy(nmPrivateTokensPath, tokenDst);
+
+
+
+ String localDirMount = toMount(localDirs);
+ String logDirMount = toMount(logDirs);
+ String containerWorkDirMount = toMount(Collections.singletonList(containerWorkDir.toUri().getPath()));
+ StringBuilder commands = new StringBuilder();
+ String commandStr = commands.append(dockerExecutor)
+ .append(" ")
+ .append("run")
+ .append(" ")
+ .append("--rm --net=host")
+ .append(" ")
+ .append(" --name " + containerIdStr)
+ .append(localDirMount)
+ .append(logDirMount)
+ .append(containerWorkDirMount)
+ .append(" ")
+ .append(containerImageName)
+ .toString();
+ String dockerPidScript = "`" + dockerExecutor + " inspect --format {{.State.Pid}} " + containerIdStr + "`";
+ // Create new local launch wrapper script
+ LocalWrapperScriptBuilder sb =
+ new UnixLocalWrapperScriptBuilder(containerWorkDir, commandStr, dockerPidScript);
+ Path pidFile = getPidFilePath(containerId);
+ if (pidFile != null) {
+ sb.writeLocalWrapperScript(launchDst, pidFile);
+ } else {
+ LOG.info("Container " + containerIdStr
+ + " was marked as inactive. Returning terminated error");
+ return ExitCode.TERMINATED.getExitCode();
+ }
+
+ ShellCommandExecutor shExec = null;
+ try {
+ lfs.setPermission(launchDst,
+ ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);
+ lfs.setPermission(sb.getWrapperScriptPath(),
+ ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);
+
+ // Setup command to run
+ String[] command = getRunCommand(sb.getWrapperScriptPath().toString(),
+ containerIdStr, userName, pidFile, this.getConf());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("launchContainer: " + commandStr + " " + Joiner.on(" ").join(command));
+ }
+ shExec = new ShellCommandExecutor(
+ command,
+ new File(containerWorkDir.toUri().getPath()),
+ container.getLaunchContext().getEnvironment()); // sanitized env
+ if (isContainerActive(containerId)) {
+ shExec.execute();
+ } else {
+ LOG.info("Container " + containerIdStr +
+ " was marked as inactive. Returning terminated error");
+ return ExitCode.TERMINATED.getExitCode();
+ }
+ } catch (IOException e) {
+ if (null == shExec) {
+ return -1;
+ }
+ int exitCode = shExec.getExitCode();
+ LOG.warn("Exit code from container " + containerId + " is : " + exitCode);
+ // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the container was
+ // terminated/killed forcefully. In all other cases, log the
+ // container-executor's output
+ if (exitCode != ExitCode.FORCE_KILLED.getExitCode()
+ && exitCode != ExitCode.TERMINATED.getExitCode()) {
+ LOG.warn("Exception from container-launch with container ID: "
+ + containerId + " and exit code: " + exitCode, e);
+ logOutput(shExec.getOutput());
+ String diagnostics = "Exception from container-launch: \n"
+ + StringUtils.stringifyException(e) + "\n" + shExec.getOutput();
+ container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
+ diagnostics));
+ } else {
+ container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
+ "Container killed on request. Exit code is " + exitCode));
+ }
+ return exitCode;
+ } finally {
+ if (shExec != null) {
+ shExec.close();
+ }
+ }
+ return 0;
+ }
+
+ @Override
+ public void writeLaunchEnv(OutputStream out, Map<String, String> environment, Map<Path, List<String>> resources, List<String> command) throws IOException {
+ ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder.create();
+
+ Set<String> exclusionSet = new HashSet<String>();
+ exclusionSet.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME);
+ exclusionSet.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name());
+ exclusionSet.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name());
+ exclusionSet.add(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name());
+ exclusionSet.add(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
+ exclusionSet.add(ApplicationConstants.Environment.JAVA_HOME.name());
+
+ if (environment != null) {
+ for (Map.Entry<String,String> env : environment.entrySet()) {
+ if (!exclusionSet.contains(env.getKey())) {
+ sb.env(env.getKey().toString(), env.getValue().toString());
+ }
+ }
+ }
+ if (resources != null) {
+ for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
+ for (String linkName : entry.getValue()) {
+ sb.symlink(entry.getKey(), new Path(linkName));
+ }
+ }
+ }
+
+ sb.command(command);
+
+ PrintStream pout = null;
+ PrintStream ps = null;
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ pout = new PrintStream(out);
+ if (LOG.isDebugEnabled()) {
+ ps = new PrintStream(baos);
+ sb.write(ps);
+ }
+ sb.write(pout);
+
+ } finally {
+ if (out != null) {
+ out.close();
+ }
+ if (ps != null) {
+ ps.close();
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Script: " + baos.toString());
+ }
+ }
+
+ private boolean saneDockerImage(String containerImageName) {
+ return dockerImagePattern.matcher(containerImageName).matches();
+ }
+
+ @Override
+ public boolean signalContainer(String user, String pid, Signal signal)
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sending signal " + signal.getValue() + " to pid " + pid
+ + " as user " + user);
+ }
+ if (!containerIsAlive(pid)) {
+ return false;
+ }
+ try {
+ killContainer(pid, signal);
+ } catch (IOException e) {
+ if (!containerIsAlive(pid)) {
+ return false;
+ }
+ throw e;
+ }
+ return true;
+ }
+
+ @Override
+ public boolean isContainerProcessAlive(String user, String pid)
+ throws IOException {
+ return containerIsAlive(pid);
+ }
+
+ /**
+ * Returns true if the process with the specified pid is alive.
+ *
+ * @param pid String pid
+ * @return boolean true if the process is alive
+ */
+ @VisibleForTesting
+ public static boolean containerIsAlive(String pid) throws IOException {
+ try {
+ new ShellCommandExecutor(Shell.getCheckProcessIsAliveCommand(pid))
+ .execute();
+ // successful execution means process is alive
+ return true;
+ }
+ catch (Shell.ExitCodeException e) {
+ // failure (non-zero exit code) means process is not alive
+ return false;
+ }
+ }
+
+ /**
+ * Send a specified signal to the specified pid
+ *
+ * @param pid the pid of the process [group] to signal.
+ * @param signal signal to send
+ * (for logging).
+ */
+ protected void killContainer(String pid, Signal signal) throws IOException {
+ new ShellCommandExecutor(Shell.getSignalKillCommand(signal.getValue(), pid))
+ .execute();
+ }
+
+ @Override
+ public void deleteAsUser(String user, Path subDir, Path... baseDirs)
+ throws IOException, InterruptedException {
+ if (baseDirs == null || baseDirs.length == 0) {
+ LOG.info("Deleting absolute path : " + subDir);
+ if (!lfs.delete(subDir, true)) {
+ //Maybe retry
+ LOG.warn("delete returned false for path: [" + subDir + "]");
+ }
+ return;
+ }
+ for (Path baseDir : baseDirs) {
+ Path del = subDir == null ? baseDir : new Path(baseDir, subDir);
+ LOG.info("Deleting path : " + del);
+ if (!lfs.delete(del, true)) {
+ LOG.warn("delete returned false for path: [" + del + "]");
+ }
+ }
+ }
+
+ /**
+ * Converts a directory list to a docker mount string
+ * @param dirs
+ * @return a string of mounts for docker
+ */
+ private String toMount(List<String> dirs) {
+ StringBuilder builder = new StringBuilder();
+ for (String dir : dirs) {
+ builder.append(" -v " + dir + ":" + dir);
+ }
+ return builder.toString();
+ }
+
+ private abstract class LocalWrapperScriptBuilder {
+
+ private final Path wrapperScriptPath;
+
+ public Path getWrapperScriptPath() {
+ return wrapperScriptPath;
+ }
+
+ public void writeLocalWrapperScript(Path launchDst, Path pidFile) throws IOException {
+ DataOutputStream out = null;
+ PrintStream pout = null;
+
+ try {
+ out = lfs.create(wrapperScriptPath, EnumSet.of(CREATE, OVERWRITE));
+ pout = new PrintStream(out);
+ writeLocalWrapperScript(launchDst, pidFile, pout);
+ } finally {
+ IOUtils.cleanup(LOG, pout, out);
+ }
+ }
+
+ protected abstract void writeLocalWrapperScript(Path launchDst, Path pidFile,
+ PrintStream pout);
+
+ protected LocalWrapperScriptBuilder(Path containerWorkDir) {
+ this.wrapperScriptPath = new Path(containerWorkDir,
+ Shell.appendScriptExtension(DOCKER_CONTAINER_EXECUTOR_SCRIPT));
+ }
+ }
+
+ private final class UnixLocalWrapperScriptBuilder
+ extends LocalWrapperScriptBuilder {
+ private final Path sessionScriptPath;
+ private final String dockerCommand;
+ private final String dockerPidScript;
+
+ public UnixLocalWrapperScriptBuilder(Path containerWorkDir, String dockerCommand, String dockerPidScript) {
+ super(containerWorkDir);
+ this.dockerCommand = dockerCommand;
+ this.dockerPidScript = dockerPidScript;
+ this.sessionScriptPath = new Path(containerWorkDir,
+ Shell.appendScriptExtension(DOCKER_CONTAINER_EXECUTOR_SESSION_SCRIPT));
+ }
+
+ @Override
+ public void writeLocalWrapperScript(Path launchDst, Path pidFile)
+ throws IOException {
+ writeSessionScript(launchDst, pidFile);
+ super.writeLocalWrapperScript(launchDst, pidFile);
+ }
+
+ @Override
+ public void writeLocalWrapperScript(Path launchDst, Path pidFile,
+ PrintStream pout) {
+
+ String exitCodeFile = ContainerLaunch.getExitCodeFile(
+ pidFile.toString());
+ String tmpFile = exitCodeFile + ".tmp";
+ pout.println("#!/usr/bin/env bash");
+ pout.println("bash \"" + sessionScriptPath.toString() + "\"");
+ pout.println("rc=$?");
+ pout.println("echo $rc > \"" + tmpFile + "\"");
+ pout.println("mv -f \"" + tmpFile + "\" \"" + exitCodeFile + "\"");
+ pout.println("exit $rc");
+ }
+
+ private void writeSessionScript(Path launchDst, Path pidFile)
+ throws IOException {
+ DataOutputStream out = null;
+ PrintStream pout = null;
+ try {
+ out = lfs.create(sessionScriptPath, EnumSet.of(CREATE, OVERWRITE));
+ pout = new PrintStream(out);
+ // We need to do a move as writing to a file is not atomic
+ // Process reading a file being written to may get garbled data
+ // hence write pid to tmp file first followed by a mv
+ pout.println("#!/usr/bin/env bash");
+ pout.println();
+ pout.println("echo "+ dockerPidScript +" > " + pidFile.toString() + ".tmp");
+ pout.println("/bin/mv -f " + pidFile.toString() + ".tmp " + pidFile);
+ pout.println(dockerCommand + " bash \"" +
+ launchDst.toUri().getPath().toString() + "\"");
+ } finally {
+ IOUtils.cleanup(LOG, pout, out);
+ }
+ lfs.setPermission(sessionScriptPath,
+ ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);
+ }
+ }
+
+ protected void createDir(Path dirPath, FsPermission perms,
+ boolean createParent, String user) throws IOException {
+ lfs.mkdir(dirPath, perms, createParent);
+ if (!perms.equals(perms.applyUMask(lfs.getUMask()))) {
+ lfs.setPermission(dirPath, perms);
+ }
+ }
+
+ /**
+ * Initialize the local directories for a particular user.
+ * <ul>.mkdir
+ * <li>$local.dir/usercache/$user</li>
+ * </ul>
+ */
+ void createUserLocalDirs(List<String> localDirs, String user)
+ throws IOException {
+ boolean userDirStatus = false;
+ FsPermission userperms = new FsPermission(USER_PERM);
+ for (String localDir : localDirs) {
+ // create $local.dir/usercache/$user and its immediate parent
+ try {
+ createDir(getUserCacheDir(new Path(localDir), user), userperms, true, user);
+ } catch (IOException e) {
+ LOG.warn("Unable to create the user directory : " + localDir, e);
+ continue;
+ }
+ userDirStatus = true;
+ }
+ if (!userDirStatus) {
+ throw new IOException("Not able to initialize user directories "
+ + "in any of the configured local directories for user " + user);
+ }
+ }
+
+
+ /**
+ * Initialize the local cache directories for a particular user.
+ * <ul>
+ * <li>$local.dir/usercache/$user</li>
+ * <li>$local.dir/usercache/$user/appcache</li>
+ * <li>$local.dir/usercache/$user/filecache</li>
+ * </ul>
+ */
+ void createUserCacheDirs(List<String> localDirs, String user)
+ throws IOException {
+ LOG.info("Initializing user " + user);
+
+ boolean appcacheDirStatus = false;
+ boolean distributedCacheDirStatus = false;
+ FsPermission appCachePerms = new FsPermission(APPCACHE_PERM);
+ FsPermission fileperms = new FsPermission(FILECACHE_PERM);
+
+ for (String localDir : localDirs) {
+ // create $local.dir/usercache/$user/appcache
+ Path localDirPath = new Path(localDir);
+ final Path appDir = getAppcacheDir(localDirPath, user);
+ try {
+ createDir(appDir, appCachePerms, true, user);
+ appcacheDirStatus = true;
+ } catch (IOException e) {
+ LOG.warn("Unable to create app cache directory : " + appDir, e);
+ }
+ // create $local.dir/usercache/$user/filecache
+ final Path distDir = getFileCacheDir(localDirPath, user);
+ try {
+ createDir(distDir, fileperms, true, user);
+ distributedCacheDirStatus = true;
+ } catch (IOException e) {
+ LOG.warn("Unable to create file cache directory : " + distDir, e);
+ }
+ }
+ if (!appcacheDirStatus) {
+ throw new IOException("Not able to initialize app-cache directories "
+ + "in any of the configured local directories for user " + user);
+ }
+ if (!distributedCacheDirStatus) {
+ throw new IOException(
+ "Not able to initialize distributed-cache directories "
+ + "in any of the configured local directories for user "
+ + user);
+ }
+ }
+
+ /**
+ * Initialize the local directories for a particular user.
+ * <ul>
+ * <li>$local.dir/usercache/$user/appcache/$appid</li>
+ * </ul>
+ * @param localDirs
+ */
+ void createAppDirs(List<String> localDirs, String user, String appId)
+ throws IOException {
+ boolean initAppDirStatus = false;
+ FsPermission appperms = new FsPermission(APPDIR_PERM);
+ for (String localDir : localDirs) {
+ Path fullAppDir = getApplicationDir(new Path(localDir), user, appId);
+ // create $local.dir/usercache/$user/appcache/$appId
+ try {
+ createDir(fullAppDir, appperms, true, user);
+ initAppDirStatus = true;
+ } catch (IOException e) {
+ LOG.warn("Unable to create app directory " + fullAppDir.toString(), e);
+ }
+ }
+ if (!initAppDirStatus) {
+ throw new IOException("Not able to initialize app directories "
+ + "in any of the configured local directories for app "
+ + appId.toString());
+ }
+ }
+
+
+ /**
+ * Create application log directories on all disks.
+ */
+ void createContainerLogDirs(String appId, String containerId,
+ List<String> logDirs, String user) throws IOException {
+
+ boolean containerLogDirStatus = false;
+ FsPermission containerLogDirPerms = new FsPermission(LOGDIR_PERM);
+ for (String rootLogDir : logDirs) {
+ // create $log.dir/$appid/$containerid
+ Path appLogDir = new Path(rootLogDir, appId);
+ Path containerLogDir = new Path(appLogDir, containerId);
+ try {
+ createDir(containerLogDir, containerLogDirPerms, true, user);
+ } catch (IOException e) {
+ LOG.warn("Unable to create the container-log directory : "
+ + appLogDir, e);
+ continue;
+ }
+ containerLogDirStatus = true;
+ }
+ if (!containerLogDirStatus) {
+ throw new IOException(
+ "Not able to initialize container-log directories "
+ + "in any of the configured local directories for container "
+ + containerId);
+ }
+ }
+
+ /**
+ * Permissions for user dir.
+ * $local.dir/usercache/$user
+ */
+ static final short USER_PERM = (short) 0750;
+ /**
+ * Permissions for user appcache dir.
+ * $local.dir/usercache/$user/appcache
+ */
+ static final short APPCACHE_PERM = (short) 0710;
+ /**
+ * Permissions for user filecache dir.
+ * $local.dir/usercache/$user/filecache
+ */
+ static final short FILECACHE_PERM = (short) 0710;
+ /**
+ * Permissions for user app dir.
+ * $local.dir/usercache/$user/appcache/$appId
+ */
+ static final short APPDIR_PERM = (short) 0710;
+ /**
+ * Permissions for user log dir.
+ * $logdir/$user/$appId
+ */
+ static final short LOGDIR_PERM = (short) 0710;
+
+ private long getDiskFreeSpace(Path base) throws IOException {
+ return lfs.getFsStatus(base).getRemaining();
+ }
+
+ private Path getApplicationDir(Path base, String user, String appId) {
+ return new Path(getAppcacheDir(base, user), appId);
+ }
+
+ private Path getUserCacheDir(Path base, String user) {
+ return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
+ }
+
+ private Path getAppcacheDir(Path base, String user) {
+ return new Path(getUserCacheDir(base, user),
+ ContainerLocalizer.APPCACHE);
+ }
+
+ private Path getFileCacheDir(Path base, String user) {
+ return new Path(getUserCacheDir(base, user),
+ ContainerLocalizer.FILECACHE);
+ }
+
+ protected Path getWorkingDir(List<String> localDirs, String user,
+ String appId) throws IOException {
+ Path appStorageDir = null;
+ long totalAvailable = 0L;
+ long[] availableOnDisk = new long[localDirs.size()];
+ int i = 0;
+ // randomly choose the app directory
+ // the chance of picking a directory is proportional to
+ // the available space on the directory.
+ // firstly calculate the sum of all available space on these directories
+ for (String localDir : localDirs) {
+ Path curBase = getApplicationDir(new Path(localDir),
+ user, appId);
+ long space = 0L;
+ try {
+ space = getDiskFreeSpace(curBase);
+ } catch (IOException e) {
+ LOG.warn("Unable to get Free Space for " + curBase.toString(), e);
+ }
+ availableOnDisk[i++] = space;
+ totalAvailable += space;
+ }
+
+ // throw an IOException if totalAvailable is 0.
+ if (totalAvailable <= 0L) {
+ throw new IOException("Not able to find a working directory for "
+ + user);
+ }
+
+ // make probability to pick a directory proportional to
+ // the available space on the directory.
+ Random r = new Random();
+ long randomPosition = Math.abs(r.nextLong()) % totalAvailable;
+ int dir = 0;
+ // skip zero available space directory,
+ // because totalAvailable is greater than 0 and randomPosition
+ // is less than totalAvailable, we can find a valid directory
+ // with nonzero available space.
+ while (availableOnDisk[dir] == 0L) {
+ dir++;
+ }
+ while (randomPosition > availableOnDisk[dir]) {
+ randomPosition -= availableOnDisk[dir++];
+ }
+ appStorageDir = getApplicationDir(new Path(localDirs.get(dir)),
+ user, appId);
+
+ return appStorageDir;
+ }
+
+ /**
+ * Create application log directories on all disks.
+ */
+ void createAppLogDirs(String appId, List<String> logDirs, String user)
+ throws IOException {
+
+ boolean appLogDirStatus = false;
+ FsPermission appLogDirPerms = new FsPermission(LOGDIR_PERM);
+ for (String rootLogDir : logDirs) {
+ // create $log.dir/$appid
+ Path appLogDir = new Path(rootLogDir, appId);
+ try {
+ createDir(appLogDir, appLogDirPerms, true, user);
+ } catch (IOException e) {
+ LOG.warn("Unable to create the app-log directory : " + appLogDir, e);
+ continue;
+ }
+ appLogDirStatus = true;
+ }
+ if (!appLogDirStatus) {
+ throw new IOException("Not able to initialize app-log directories "
+ + "in any of the configured local directories for app " + appId);
+ }
+ }
+
+ /**
+ * @return the list of paths of given local directories
+ */
+ private static List<Path> getPaths(List<String> dirs) {
+ List<Path> paths = new ArrayList<Path>(dirs.size());
+ for (int i = 0; i < dirs.size(); i++) {
+ paths.add(new Path(dirs.get(i)));
+ }
+ return paths;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 8fc5ea3..a87238d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -269,7 +269,7 @@ public class ContainerLaunch implements Callable<Integer> {
localResources, nmPrivateClasspathJarDir);
// Write out the environment
- writeLaunchEnv(containerScriptOutStream, environment, localResources,
+ exec.writeLaunchEnv(containerScriptOutStream, environment, localResources,
launchContext.getCommands());
// /////////// End of writing out container-script
@@ -502,8 +502,7 @@ public class ContainerLaunch implements Callable<Integer> {
return context;
}
- @VisibleForTesting
- static abstract class ShellScriptBuilder {
+ public static abstract class ShellScriptBuilder {
public static ShellScriptBuilder create() {
return Shell.WINDOWS ? new WindowsShellScriptBuilder() :
new UnixShellScriptBuilder();
@@ -793,37 +792,6 @@ public class ContainerLaunch implements Callable<Integer> {
meta.getKey(), meta.getValue(), environment);
}
}
-
- static void writeLaunchEnv(OutputStream out,
- Map<String,String> environment, Map<Path,List<String>> resources,
- List<String> command)
- throws IOException {
- ShellScriptBuilder sb = ShellScriptBuilder.create();
- if (environment != null) {
- for (Map.Entry<String,String> env : environment.entrySet()) {
- sb.env(env.getKey().toString(), env.getValue().toString());
- }
- }
- if (resources != null) {
- for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
- for (String linkName : entry.getValue()) {
- sb.symlink(entry.getKey(), new Path(linkName));
- }
- }
- }
-
- sb.command(command);
-
- PrintStream pout = null;
- try {
- pout = new PrintStream(out);
- sb.write(pout);
- } finally {
- if (out != null) {
- out.close();
- }
- }
- }
public static String getExitCodeFile(String pidFile) {
return pidFile + EXIT_CODE_FILE_SUFFIX;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java
new file mode 100644
index 0000000..e43ac2e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import com.google.common.base.Strings;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * This is intended to test the DockerContainerExecutor code, but it requires docker
+ * to be installed.
+ * <br><ol>
+ * <li>Install docker, and Compile the code with docker-service-url set to the host and port
+ * where docker service is running.
+ * <br><pre><code>
+ * > mvn clean install -Ddocker-service-url=tcp://0.0.0.0:4243
+ * -DskipTests
+ * </code></pre>
+ */
+public class TestDockerContainerExecutor {
+ private static final Log LOG = LogFactory
+ .getLog(TestDockerContainerExecutor.class);
+ private static File workSpace = null;
+ private DockerContainerExecutor exec = null;
+ private LocalDirsHandlerService dirsHandler;
+ private Path workDir;
+ private FileContext lfs;
+ private String yarnImage;
+
+ private int id = 0;
+ private String appSubmitter;
+ private String dockerUrl;
+ private String testImage = "centos";
+ private String dockerExec;
+ private String containerIdStr;
+
+
+ private ContainerId getNextContainerId() {
+ ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
+ String id = "CONTAINER_" + System.currentTimeMillis();
+ when(cId.toString()).thenReturn(id);
+ return cId;
+ }
+
+ @Before
+ public void setup() {
+ try {
+ lfs = FileContext.getLocalFSFileContext();
+ workDir = new Path("/tmp/temp-" + System.currentTimeMillis());
+ workSpace = new File(workDir.toUri().getPath());
+ lfs.mkdir(workDir, FsPermission.getDirDefault(), true);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Configuration conf = new Configuration();
+ yarnImage = "yarnImage";
+ long time = System.currentTimeMillis();
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, "/tmp/nm-local-dir" + time);
+ conf.set(YarnConfiguration.NM_LOG_DIRS, "/tmp/userlogs" + time);
+
+ dockerUrl = System.getProperty("docker-service-url");
+ LOG.info("dockerUrl: " + dockerUrl);
+ if (Strings.isNullOrEmpty(dockerUrl)) {
+ return;
+ }
+ dockerUrl = " -H " + dockerUrl;
+ dockerExec = "docker " + dockerUrl;
+ conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, yarnImage);
+ conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME, dockerExec);
+ exec = new DockerContainerExecutor();
+ dirsHandler = new LocalDirsHandlerService();
+ dirsHandler.init(conf);
+ exec.setConf(conf);
+ appSubmitter = System.getProperty("application.submitter");
+ if (appSubmitter == null || appSubmitter.isEmpty()) {
+ appSubmitter = "nobody";
+ }
+ shellExec(dockerExec + " pull " + testImage);
+
+ }
+
+ private Shell.ShellCommandExecutor shellExec(String command) {
+ try {
+
+ Shell.ShellCommandExecutor shExec = new Shell.ShellCommandExecutor(
+ command.split("\\s+"),
+ new File(workDir.toUri().getPath()),
+ System.getenv());
+ shExec.execute();
+ return shExec;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private boolean shouldRun() {
+ return exec != null;
+ }
+
+ private int runAndBlock(ContainerId cId, Map<String, String> launchCtxEnv, String... cmd) throws IOException {
+ String appId = "APP_" + System.currentTimeMillis();
+ Container container = mock(Container.class);
+ ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+
+ when(container.getContainerId()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(context);
+ when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId);
+ when(context.getEnvironment()).thenReturn(launchCtxEnv);
+
+ String script = writeScriptFile(launchCtxEnv, cmd);
+
+ Path scriptPath = new Path(script);
+ Path tokensPath = new Path("/dev/null");
+ Path workDir = new Path(workSpace.getAbsolutePath());
+ Path pidFile = new Path(workDir, "pid.txt");
+
+ exec.activateContainer(cId, pidFile);
+ return exec.launchContainer(container, scriptPath, tokensPath,
+ appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
+ dirsHandler.getLogDirs());
+ }
+
+ private String writeScriptFile(Map<String, String> launchCtxEnv, String... cmd) throws IOException {
+ File f = File.createTempFile("TestDockerContainerExecutor", ".sh");
+ f.deleteOnExit();
+ PrintWriter p = new PrintWriter(new FileOutputStream(f));
+ for(Map.Entry<String, String> entry: launchCtxEnv.entrySet()) {
+ p.println("export " + entry.getKey() + "=\"" + entry.getValue() + "\"");
+ }
+ for (String part : cmd) {
+ p.print(part.replace("\\", "\\\\").replace("'", "\\'"));
+ p.print(" ");
+ }
+ p.println();
+ p.close();
+ return f.getAbsolutePath();
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ lfs.delete(workDir, true);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test
+ public void testLaunchContainer() throws IOException {
+ if (!shouldRun()) {
+ LOG.warn("Docker not installed, aborting test.");
+ return;
+ }
+
+ Map<String, String> env = new HashMap<String, String>();
+ env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
+ String touchFileName = "touch-file-" + System.currentTimeMillis();
+ File touchFile = new File(dirsHandler.getLocalDirs().get(0), touchFileName);
+ ContainerId cId = getNextContainerId();
+ int ret = runAndBlock(
+ cId, env, "touch", touchFile.getAbsolutePath(), "&&", "cp", touchFile.getAbsolutePath(), "/");
+
+ assertEquals(0, ret);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
new file mode 100644
index 0000000..fa8bfaf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import com.google.common.base.Strings;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+
+/**
+ * Mock tests for docker container executor
+ */
+public class TestDockerContainerExecutorWithMocks {
+
+ private static final Log LOG = LogFactory
+ .getLog(TestDockerContainerExecutorWithMocks.class);
+ public static final String DOCKER_LAUNCH_COMMAND = "/bin/true";
+ private DockerContainerExecutor dockerContainerExecutor = null;
+ private LocalDirsHandlerService dirsHandler;
+ private Path workDir;
+ private FileContext lfs;
+ private String yarnImage;
+
+ @Before
+ public void setup() {
+ assumeTrue(!Path.WINDOWS);
+ File f = new File("./src/test/resources/mock-container-executor");
+ if(!FileUtil.canExecute(f)) {
+ FileUtil.setExecutable(f, true);
+ }
+ String executorPath = f.getAbsolutePath();
+ Configuration conf = new Configuration();
+ yarnImage = "yarnImage";
+ long time = System.currentTimeMillis();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, "/tmp/nm-local-dir" + time);
+ conf.set(YarnConfiguration.NM_LOG_DIRS, "/tmp/userlogs" + time);
+ conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, yarnImage);
+ conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME , DOCKER_LAUNCH_COMMAND);
+ dockerContainerExecutor = new DockerContainerExecutor();
+ dirsHandler = new LocalDirsHandlerService();
+ dirsHandler.init(conf);
+ dockerContainerExecutor.setConf(conf);
+ lfs = null;
+ try {
+ lfs = FileContext.getLocalFSFileContext();
+ workDir = new Path("/tmp/temp-"+ System.currentTimeMillis());
+ lfs.mkdir(workDir, FsPermission.getDirDefault(), true);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ lfs.delete(workDir, true);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testContainerInitSecure() throws IOException {
+ dockerContainerExecutor.getConf().set(
+ CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ dockerContainerExecutor.init();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testContainerLaunchNullImage() throws IOException {
+ String appSubmitter = "nobody";
+ String appId = "APP_ID";
+ String containerId = "CONTAINER_ID";
+ String testImage = "";
+
+ Container container = mock(Container.class, RETURNS_DEEP_STUBS);
+ ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
+ ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+ HashMap<String, String> env = new HashMap<String,String>();
+
+ when(container.getContainerId()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(context);
+ when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId);
+ when(cId.toString()).thenReturn(containerId);
+
+ when(context.getEnvironment()).thenReturn(env);
+ env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
+ dockerContainerExecutor.getConf()
+ .set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
+ Path scriptPath = new Path("file:///bin/echo");
+ Path tokensPath = new Path("file:///dev/null");
+
+ Path pidFile = new Path(workDir, "pid.txt");
+
+ dockerContainerExecutor.activateContainer(cId, pidFile);
+ dockerContainerExecutor.launchContainer(container, scriptPath, tokensPath,
+ appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
+ dirsHandler.getLogDirs());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testContainerLaunchInvalidImage() throws IOException {
+ String appSubmitter = "nobody";
+ String appId = "APP_ID";
+ String containerId = "CONTAINER_ID";
+ String testImage = "testrepo.com/test-image rm -rf $HADOOP_PREFIX/*";
+
+ Container container = mock(Container.class, RETURNS_DEEP_STUBS);
+ ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
+ ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+ HashMap<String, String> env = new HashMap<String,String>();
+
+ when(container.getContainerId()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(context);
+ when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId);
+ when(cId.toString()).thenReturn(containerId);
+
+ when(context.getEnvironment()).thenReturn(env);
+ env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
+ dockerContainerExecutor.getConf()
+ .set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
+ Path scriptPath = new Path("file:///bin/echo");
+ Path tokensPath = new Path("file:///dev/null");
+
+ Path pidFile = new Path(workDir, "pid.txt");
+
+ dockerContainerExecutor.activateContainer(cId, pidFile);
+ dockerContainerExecutor.launchContainer(container, scriptPath, tokensPath,
+ appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
+ dirsHandler.getLogDirs());
+ }
+
+ @Test
+ public void testContainerLaunch() throws IOException {
+ String appSubmitter = "nobody";
+ String appId = "APP_ID";
+ String containerId = "CONTAINER_ID";
+ String testImage = "\"sequenceiq/hadoop-docker:2.4.1\"";
+
+ Container container = mock(Container.class, RETURNS_DEEP_STUBS);
+ ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
+ ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+ HashMap<String, String> env = new HashMap<String,String>();
+
+ when(container.getContainerId()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(context);
+ when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId);
+ when(cId.toString()).thenReturn(containerId);
+
+ when(context.getEnvironment()).thenReturn(env);
+ env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
+ Path scriptPath = new Path("file:///bin/echo");
+ Path tokensPath = new Path("file:///dev/null");
+
+ Path pidFile = new Path(workDir, "pid");
+
+ dockerContainerExecutor.activateContainer(cId, pidFile);
+ int ret = dockerContainerExecutor.launchContainer(container, scriptPath, tokensPath,
+ appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
+ dirsHandler.getLogDirs());
+ assertEquals(0, ret);
+ //get the script
+ Path sessionScriptPath = new Path(workDir,
+ Shell.appendScriptExtension(
+ DockerContainerExecutor.DOCKER_CONTAINER_EXECUTOR_SESSION_SCRIPT));
+ LineNumberReader lnr = new LineNumberReader(new FileReader(sessionScriptPath.toString()));
+ boolean cmdFound = false;
+ List<String> localDirs = dirsToMount(dirsHandler.getLocalDirs());
+ List<String> logDirs = dirsToMount(dirsHandler.getLogDirs());
+ List<String> workDirMount = dirsToMount(Collections.singletonList(workDir.toUri().getPath()));
+ List<String> expectedCommands = new ArrayList<String>(
+ Arrays.asList(DOCKER_LAUNCH_COMMAND, "run", "--rm", "--net=host", "--name", containerId));
+ expectedCommands.addAll(localDirs);
+ expectedCommands.addAll(logDirs);
+ expectedCommands.addAll(workDirMount);
+ String shellScript = workDir + "/launch_container.sh";
+
+ expectedCommands.addAll(Arrays.asList(testImage.replaceAll("['\"]", ""), "bash","\"" + shellScript + "\""));
+
+ String expectedPidString = "echo `/bin/true inspect --format {{.State.Pid}} " + containerId+"` > "+ pidFile.toString() + ".tmp";
+ boolean pidSetterFound = false;
+ while(lnr.ready()){
+ String line = lnr.readLine();
+ LOG.debug("line: " + line);
+ if (line.startsWith(DOCKER_LAUNCH_COMMAND)){
+ List<String> command = new ArrayList<String>();
+ for( String s :line.split("\\s+")){
+ command.add(s.trim());
+ }
+
+ assertEquals(expectedCommands, command);
+ cmdFound = true;
+ } else if (line.startsWith("echo")) {
+ assertEquals(expectedPidString, line);
+ pidSetterFound = true;
+ }
+
+ }
+ assertTrue(cmdFound);
+ assertTrue(pidSetterFound);
+ }
+
+ private List<String> dirsToMount(List<String> dirs) {
+ List<String> localDirs = new ArrayList<String>();
+ for(String dir: dirs){
+ localDirs.add("-v");
+ localDirs.add(dir + ":" + dir);
+ }
+ return localDirs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index cbc41c4..4088c2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -144,7 +145,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
commands.add("/bin/sh ./\\\"" + badSymlink + "\\\"");
}
- ContainerLaunch.writeLaunchEnv(fos, env, resources, commands);
+ new DefaultContainerExecutor().writeLaunchEnv(fos, env, resources, commands);
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -211,7 +212,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
} else {
commands.add("/bin/sh ./\\\"" + symLink + "\\\"");
}
- ContainerLaunch.writeLaunchEnv(fos, env, resources, commands);
+ new DefaultContainerExecutor().writeLaunchEnv(fos, env, resources, commands);
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -264,7 +265,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
"\"workflowName\":\"\n\ninsert table " +
"\npartition (cd_education_status)\nselect cd_demo_sk, cd_gender, " );
List<String> commands = new ArrayList<String>();
- ContainerLaunch.writeLaunchEnv(fos, env, resources, commands);
+ new DefaultContainerExecutor().writeLaunchEnv(fos, env, resources, commands);
fos.flush();
fos.close();
@@ -341,7 +342,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Map<String, String> env = new HashMap<String, String>();
List<String> commands = new ArrayList<String>();
commands.add(command);
- ContainerLaunch.writeLaunchEnv(fos, env, resources, commands);
+ ContainerExecutor exec = new DefaultContainerExecutor();
+ exec.writeLaunchEnv(fos, env, resources, commands);
fos.flush();
fos.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53f64ee5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/DockerContainerExecutor.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/DockerContainerExecutor.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/DockerContainerExecutor.apt.vm
new file mode 100644
index 0000000..4573173
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/DockerContainerExecutor.apt.vm
@@ -0,0 +1,200 @@
+
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+ ---
+ Hadoop Map Reduce Next Generation-${project.version} - Docker Container Executor
+ ---
+ ---
+ ${maven.build.timestamp}
+
+Docker Container Executor
+
+%{toc|section=1|fromDepth=0}
+
+* {Overview}
+
+ Docker (https://www.docker.io/) combines an easy-to-use interface to
+Linux containers with easy-to-construct image files for those
+containers. In short, Docker launches very light weight virtual
+machines.
+
+ The Docker Container Executor (DCE) allows the YARN NodeManager to
+launch YARN containers into Docker containers. Users can specify the
+Docker images they want for their YARN containers. These containers
+provide a custom software environment in which the user's code runs,
+isolated from the software environment of the NodeManager. These
+containers can include special libraries needed by the application,
+and they can have different versions of Perl, Python, and even Java
+than what is installed on the NodeManager. Indeed, these containers
+can run a different flavor of Linux than what is running on the
+NodeManager -- although the YARN container must define all the environments
+ and libraries needed to run the job, nothing will be shared with the NodeManager.
+
+ Docker for YARN provides both consistency (all YARN containers will
+have the same software environment) and isolation (no interference
+with whatever is installed on the physical machine).
+
+* {Cluster Configuration}
+
+ Docker Container Executor runs in non-secure mode of HDFS and
+YARN. It will not run in secure mode, and will exit if it detects
+secure mode.
+
+ The DockerContainerExecutor requires Docker daemon to be running on
+the NodeManagers, and the Docker client installed and able to start Docker
+containers. To prevent timeouts while starting jobs, the Docker
+images to be used by a job should already be downloaded in the
+NodeManagers. Here's an example of how this can be done:
+
+----
+sudo docker pull sequenceiq/hadoop-docker:2.4.1
+----
+
+ This should be done as part of the NodeManager startup.
+
+ The following properties must be set in yarn-site.xml:
+
+----
+<property>
+ <name>yarn.nodemanager.docker-container-executor.exec-name</name>
+ <value>/usr/bin/docker</value>
+ <description>
+ Name or path to the Docker client. This is a required parameter. If this is empty,
+ user must pass an image name as part of the job invocation(see below).
+ </description>
+</property>
+
+<property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.DockerContainerExecutor</value>
+ <description>
+ This is the container executor setting that ensures that all
+jobs are started with the DockerContainerExecutor.
+ </description>
+</property>
+----
+
+ Administrators should be aware that DCE doesn't currently provide
+user name-space isolation. This means, in particular, that software
+running as root in the YARN container will have root privileges in the
+underlying NodeManager. Put differently, DCE currently provides no
+better security guarantees than YARN's Default Container Executor. In
+fact, DockerContainerExecutor will exit if it detects secure yarn.
+
+* {Tips for connecting to a secure docker repository}
+
+ By default, docker images are pulled from the docker public repository. The
+format of a docker image url is: <username>/<image_name>. For example,
+sequenceiq/hadoop-docker:2.4.1 is an image in docker public repository that contains java and
+hadoop.
+
+ If you want your own private repository, you provide the repository url instead of
+your username. Therefore, the image url becomes: <private_repo_url>/<image_name>.
+For example, if your repository is on localhost:8080, your images would be like:
+ localhost:8080/hadoop-docker
+
+ To connect to a secure docker repository, you can use the following invocation:
+
+----
+docker login [OPTIONS] [SERVER]
+
+Register or log in to a Docker registry server, if no server is specified
+"https://index.docker.io/v1/" is the default.
+
+-e, --email="" Email
+-p, --password="" Password
+-u, --username="" Username
+----
+
+ If you want to login to a self-hosted registry you can specify this by adding
+the server name.
+
+----
+docker login <private_repo_url>
+----
+
+ This needs to be run as part of the NodeManager startup, or as a cron job if
+the login session expires periodically. You can login to multiple docker repositories
+from the same NodeManager, but all your users will have access to all your repositories,
+as at present the DockerContainerExecutor does not support per-job docker login.
+
+* {Job Configuration}
+
+ Currently you cannot configure any of the Docker settings with the job configuration.
+You can provide Mapper, Reducer, and ApplicationMaster environment overrides for the
+docker images, using the following 3 JVM properties respectively(only for MR jobs):
+ * mapreduce.map.env: You can override the mapper's image by passing
+yarn.nodemanager.docker-container-executor.image-name=<your_image_name>
+to this JVM property.
+ * mapreduce.reduce.env: You can override the reducer's image by passing
+yarn.nodemanager.docker-container-executor.image-name=<your_image_name>
+to this JVM property.
+ * yarn.app.mapreduce.am.env: You can override the ApplicationMaster's image
+by passing yarn.nodemanager.docker-container-executor.image-name=<your_image_name>
+to this JVM property.
+
+* {Docker Image requirements}
+
+ The Docker Images used for YARN containers must meet the following
+requirements:
+
+ The distro and version of Linux in your Docker Image can be quite different
+from that of your NodeManager. (Docker does have a few limitations in this
+regard, but you're not likely to hit them.) However, if you're using the
+MapReduce framework, then your image will need to be configured for running
+Hadoop. Java must be installed in the container, and the following environment variables
+must be defined in the image: JAVA_HOME, HADOOP_COMMON_PATH, HADOOP_HDFS_HOME,
+HADOOP_MAPRED_HOME, HADOOP_YARN_HOME, and HADOOP_CONF_DIR
+
+
+* {Working example of yarn launched docker containers.}
+
+ The following example shows how to run teragen using DockerContainerExecutor.
+
+ * First ensure that YARN is properly configured with DockerContainerExecutor(see above).
+
+----
+<property>
+ <name>yarn.nodemanager.docker-container-executor.exec-name</name>
+ <value>docker -H=tcp://0.0.0.0:4243</value>
+ <description>
+ Name or path to the Docker client. The tcp socket must be
+ where docker daemon is listening.
+ </description>
+</property>
+
+<property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.DockerContainerExecutor</value>
+ <description>
+ This is the container executor setting that ensures that all
+jobs are started with the DockerContainerExecutor.
+ </description>
+</property>
+----
+
+ * Pick a custom Docker image if you want. In this example, we'll use sequenceiq/hadoop-docker:2.4.1 from the
+docker hub repository. It has jdk, hadoop, and all the previously mentioned environment variables configured.
+ * Run:
+
+----
+hadoop jar $HADOOP_INSTALLATION_DIR/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \
+teragen \
+-Dmapreduce.map.env="yarn.nodemanager.docker-container-executor.image-name=sequenceiq/hadoop-docker:2.4.1" \
+-Dyarn.app.mapreduce.am.env="yarn.nodemanager.docker-container-executor.image-name=sequenceiq/hadoop-docker:2.4.1" \
+1000 \
+teragen_out_dir
+----
+
+ Once it succeeds, you can check the yarn debug logs to verify that docker indeed has launched containers.
+
[30/45] hadoop git commit: YARN-2766. Made ApplicationHistoryManager
return a sorted list of apps,
attempts and containers. Contributed by Robert Kanter.
Posted by vi...@apache.org.
YARN-2766. Made ApplicationHistoryManager return a sorted list of apps, attempts and containers. Contributed by Robert Kanter.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3648cb57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3648cb57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3648cb57
Branch: refs/heads/HDFS-EC
Commit: 3648cb57c9f018a3a339c26f5a0ca2779485521a
Parents: 4ae9780
Author: Zhijie Shen <zj...@apache.org>
Authored: Thu Nov 13 13:47:42 2014 -0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Thu Nov 13 13:47:42 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../ApplicationHistoryManagerOnTimelineStore.java | 7 ++++---
.../TestApplicationHistoryClientService.java | 4 ++--
3 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3648cb57/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 60f88ca..5c794bd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -81,6 +81,9 @@ Release 2.7.0 - UNRELEASED
YARN-2713. "RM Home" link in NM should point to one of the RMs in an
HA setup. (kasha)
+ YARN-2766. Made ApplicationHistoryManager return a sorted list of apps,
+ attempts and containers. (Robert Kanter via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3648cb57/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index cd429d0..c88cccc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -102,7 +103,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps =
- new HashMap<ApplicationId, ApplicationReport>();
+ new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) {
for (TimelineEntity entity : entities.getEntities()) {
ApplicationReportExt app =
@@ -128,7 +129,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
Long.MAX_VALUE, EnumSet.allOf(Field.class),
UserGroupInformation.getLoginUser());
Map<ApplicationAttemptId, ApplicationAttemptReport> appAttempts =
- new HashMap<ApplicationAttemptId, ApplicationAttemptReport>();
+ new LinkedHashMap<ApplicationAttemptId, ApplicationAttemptReport>();
for (TimelineEntity entity : entities.getEntities()) {
ApplicationAttemptReport appAttempt =
convertToApplicationAttemptReport(entity);
@@ -198,7 +199,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
UserGroupInformation.getLoginUser());
Map<ContainerId, ContainerReport> containers =
- new HashMap<ContainerId, ContainerReport>();
+ new LinkedHashMap<ContainerId, ContainerReport>();
if (entities != null && entities.getEntities() != null) {
for (TimelineEntity entity : entities.getEntities()) {
ContainerReport container = convertToContainerReport(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3648cb57/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 7c2593d..2a38d0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -168,7 +168,7 @@ public class TestApplicationHistoryClientService {
clientService.getClientHandler().getContainers(request);
List<ContainerReport> containers = response.getContainerList();
Assert.assertNotNull(containers);
- Assert.assertEquals(containerId, containers.get(1).getContainerId());
- Assert.assertEquals(containerId1, containers.get(0).getContainerId());
+ Assert.assertEquals(containerId, containers.get(0).getContainerId());
+ Assert.assertEquals(containerId1, containers.get(1).getContainerId());
}
}
[17/45] hadoop git commit: MAPREDUCE-6156. Fetcher - connect()
doesn't handle connection refused correctly. Contributed by Junping Du
Posted by vi...@apache.org.
MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused correctly. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/177e8090
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/177e8090
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/177e8090
Branch: refs/heads/HDFS-EC
Commit: 177e8090f5809beb3ebcb656cd0affbb3f487de8
Parents: 7dae5b5
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Nov 13 15:42:25 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Nov 13 15:42:25 2014 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../hadoop/mapreduce/task/reduce/Fetcher.java | 39 +++++++++++++++-----
2 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177e8090/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 96bb690..c0105e6 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -465,6 +465,9 @@ Release 2.6.0 - 2014-11-15
MAPREDUCE-5958. Wrong reduce task progress if map output is compressed
(Emilio Coppa and jlowe via kihwal)
+ MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused
+ correctly (Junping Du via jlowe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177e8090/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index 796394f..3f40853 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -407,7 +407,7 @@ class Fetcher<K,V> extends Thread {
}
if ((Time.monotonicNow() - startTime) >= this.fetchRetryTimeout) {
LOG.warn("Failed to connect to host: " + url + "after "
- + fetchRetryTimeout + "milliseconds.");
+ + fetchRetryTimeout + " milliseconds.");
throw e;
}
try {
@@ -596,7 +596,7 @@ class Fetcher<K,V> extends Thread {
} else {
// timeout, prepare to be failed.
LOG.warn("Timeout for copying MapOutput with retry on host " + host
- + "after " + fetchRetryTimeout + "milliseconds.");
+ + "after " + fetchRetryTimeout + " milliseconds.");
}
}
@@ -678,28 +678,49 @@ class Fetcher<K,V> extends Thread {
} else if (connectionTimeout > 0) {
unit = Math.min(UNIT_CONNECT_TIMEOUT, connectionTimeout);
}
+ long startTime = Time.monotonicNow();
+ long lastTime = startTime;
+ int attempts = 0;
// set the connect timeout to the unit-connect-timeout
connection.setConnectTimeout(unit);
while (true) {
try {
+ attempts++;
connection.connect();
break;
} catch (IOException ioe) {
- // update the total remaining connect-timeout
- connectionTimeout -= unit;
-
+ long currentTime = Time.monotonicNow();
+ long retryTime = currentTime - startTime;
+ long leftTime = connectionTimeout - retryTime;
+ long timeSinceLastIteration = currentTime - lastTime;
// throw an exception if we have waited for timeout amount of time
// note that the updated value if timeout is used here
- if (connectionTimeout == 0) {
+ if (leftTime <= 0) {
+ int retryTimeInSeconds = (int) retryTime/1000;
+ LOG.error("Connection retry failed with " + attempts +
+ " attempts in " + retryTimeInSeconds + " seconds");
throw ioe;
}
-
// reset the connect timeout for the last try
- if (connectionTimeout < unit) {
- unit = connectionTimeout;
+ if (leftTime < unit) {
+ unit = (int)leftTime;
// reset the connect time out for the final connect
connection.setConnectTimeout(unit);
}
+
+ if (timeSinceLastIteration < unit) {
+ try {
+ // sleep the left time of unit
+ sleep(unit - timeSinceLastIteration);
+ } catch (InterruptedException e) {
+ LOG.warn("Sleep in connection retry get interrupted.");
+ if (stopped) {
+ return;
+ }
+ }
+ }
+ // update the total remaining connect-timeout
+ lastTime = Time.monotonicNow();
}
}
}
[32/45] hadoop git commit: YARN-2856. Fixed RMAppImpl to handle
ATTEMPT_KILLED event at ACCEPTED state on app recovery. Contributed by Rohith
Sharmaks
Posted by vi...@apache.org.
YARN-2856. Fixed RMAppImpl to handle ATTEMPT_KILLED event at ACCEPTED state on app recovery. Contributed by Rohith Sharmaks
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d005404e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d005404e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d005404e
Branch: refs/heads/HDFS-EC
Commit: d005404ef7211fe96ce1801ed267a249568540fd
Parents: 1a2e5cb
Author: Jian He <ji...@apache.org>
Authored: Thu Nov 13 15:33:20 2014 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Nov 13 15:34:26 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/rmapp/RMAppImpl.java | 3 +++
.../rmapp/TestRMAppTransitions.java | 23 ++++++++++++++++++++
3 files changed, 29 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d005404e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5c794bd..4373813 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -84,6 +84,9 @@ Release 2.7.0 - UNRELEASED
YARN-2766. Made ApplicationHistoryManager return a sorted list of apps,
attempts and containers. (Robert Kanter via zjshen)
+ YARN-2856. Fixed RMAppImpl to handle ATTEMPT_KILLED event at ACCEPTED state
+ on app recovery. (Rohith Sharmaks via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d005404e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index aeb2cda..751dbe4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -229,6 +229,9 @@ public class RMAppImpl implements RMApp, Recoverable {
new FinalSavingTransition(FINISHED_TRANSITION, RMAppState.FINISHED))
.addTransition(RMAppState.ACCEPTED, RMAppState.KILLING,
RMAppEventType.KILL, new KillAttemptTransition())
+ .addTransition(RMAppState.ACCEPTED, RMAppState.FINAL_SAVING,
+ RMAppEventType.ATTEMPT_KILLED,
+ new FinalSavingTransition(new AppKilledTransition(), RMAppState.KILLED))
.addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
RMAppEventType.APP_RUNNING_ON_NODE,
new AppRunningOnNodeTransition())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d005404e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index e68d074..0a2f0d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -714,6 +714,29 @@ public class TestRMAppTransitions {
verifyApplicationFinished(RMAppState.KILLED);
verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
}
+
+ @Test
+ public void testAppAcceptedAttemptKilled() throws IOException,
+ InterruptedException {
+ LOG.info("--- START: testAppAcceptedAttemptKilled ---");
+ RMApp application = testCreateAppAccepted(null);
+
+ // ACCEPTED => FINAL_SAVING event RMAppEventType.ATTEMPT_KILLED
+ // When application recovery happens for attempt is KILLED but app is
+ // RUNNING.
+ RMAppEvent event =
+ new RMAppEvent(application.getApplicationId(),
+ RMAppEventType.ATTEMPT_KILLED);
+ application.handle(event);
+ rmDispatcher.await();
+
+ assertAppState(RMAppState.FINAL_SAVING, application);
+ sendAppUpdateSavedEvent(application);
+ assertKilled(application);
+ assertAppFinalStateSaved(application);
+ verifyApplicationFinished(RMAppState.KILLED);
+ verifyAppRemovedSchedulerEvent(RMAppState.KILLED);
+ }
@Test
public void testAppRunningKill() throws IOException {
[27/45] hadoop git commit: HDFS-7385. ThreadLocal used in FSEditLog
class causes FSImage permission mess up. Contributed by jiangyu.
Posted by vi...@apache.org.
HDFS-7385. ThreadLocal used in FSEditLog class causes FSImage permission mess up. Contributed by jiangyu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0a41de6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0a41de6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0a41de6
Branch: refs/heads/HDFS-EC
Commit: b0a41de68c5b08f534ca231293de053c0b0cbd5d
Parents: 394ba94
Author: cnauroth <cn...@apache.org>
Authored: Thu Nov 13 12:31:44 2014 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Thu Nov 13 12:34:52 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/namenode/FSEditLog.java | 2 +
.../hdfs/server/namenode/FSEditLogOp.java | 12 ++++
.../hdfs/server/namenode/TestEditLog.java | 67 ++++++++++++++++++++
4 files changed, 84 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a41de6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bd3dcd5..998a395 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1446,6 +1446,9 @@ Release 2.6.0 - 2014-11-15
HDFS-7391. Renable SSLv2Hello in HttpFS. (rkanter via acmurthy)
+ HDFS-7385. ThreadLocal used in FSEditLog class causes FSImage permission mess
+ up. (jiangyu via cnauroth)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a41de6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index e952e7c..305fcdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -708,6 +708,7 @@ public class FSEditLog implements LogsPurgeable {
Preconditions.checkArgument(newNode.isUnderConstruction());
PermissionStatus permissions = newNode.getPermissionStatus();
AddOp op = AddOp.getInstance(cache.get())
+ .reset()
.setInodeId(newNode.getId())
.setPath(path)
.setReplication(newNode.getFileReplication())
@@ -778,6 +779,7 @@ public class FSEditLog implements LogsPurgeable {
public void logMkDir(String path, INode newNode) {
PermissionStatus permissions = newNode.getPermissionStatus();
MkdirOp op = MkdirOp.getInstance(cache.get())
+ .reset()
.setInodeId(newNode.getId())
.setPath(path)
.setTimestamp(newNode.getModificationTime())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a41de6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index c9eb9fe..06b4419 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -419,6 +419,12 @@ public abstract class FSEditLogOp {
assert(opCode == OP_ADD || opCode == OP_CLOSE);
}
+ <T extends AddCloseOp> T reset() {
+ this.aclEntries = null;
+ this.xAttrs = null;
+ return (T)this;
+ }
+
<T extends AddCloseOp> T setInodeId(long inodeId) {
this.inodeId = inodeId;
return (T)this;
@@ -1410,6 +1416,12 @@ public abstract class FSEditLogOp {
return (MkdirOp)cache.get(OP_MKDIR);
}
+ MkdirOp reset() {
+ this.aclEntries = null;
+ this.xAttrs = null;
+ return this;
+ }
+
MkdirOp setInodeId(long inodeId) {
this.inodeId = inodeId;
return this;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a41de6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 7b62242..2df55aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -17,9 +17,14 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.fs.permission.AclEntryScope.*;
+import static org.apache.hadoop.fs.permission.AclEntryType.*;
+import static org.apache.hadoop.fs.permission.FsAction.*;
+import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.*;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -57,6 +62,7 @@ import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -1501,4 +1507,65 @@ public class TestEditLog {
LOG.info(String.format("loaded %d edit log segments in %.2f seconds",
NUM_EDIT_LOG_ROLLS, delta));
}
+
+ /**
+ * Edit log op instances are cached internally using thread-local storage.
+ * This test checks that the cached instances are reset in between different
+ * transactions processed on the same thread, so that we don't accidentally
+ * apply incorrect attributes to an inode.
+ *
+ * @throws IOException if there is an I/O error
+ */
+ @Test
+ public void testResetThreadLocalCachedOps() throws IOException {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ // Set single handler thread, so all transactions hit same thread-local ops.
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HANDLER_COUNT_KEY, 1);
+ MiniDFSCluster cluster = null;
+ FileSystem fileSys = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ fileSys = cluster.getFileSystem();
+
+ // Create /dir1 with a default ACL.
+ Path dir1 = new Path("/dir1");
+ fileSys.mkdirs(dir1);
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, USER, "foo", READ_EXECUTE));
+ fileSys.modifyAclEntries(dir1, aclSpec);
+
+ // /dir1/dir2 is expected to clone the default ACL.
+ Path dir2 = new Path("/dir1/dir2");
+ fileSys.mkdirs(dir2);
+
+ // /dir1/file1 is expected to clone the default ACL.
+ Path file1 = new Path("/dir1/file1");
+ fileSys.create(file1).close();
+
+ // /dir3 is not a child of /dir1, so must not clone the default ACL.
+ Path dir3 = new Path("/dir3");
+ fileSys.mkdirs(dir3);
+
+ // /file2 is not a child of /dir1, so must not clone the default ACL.
+ Path file2 = new Path("/file2");
+ fileSys.create(file2).close();
+
+ // Restart and assert the above stated expectations.
+ IOUtils.cleanup(LOG, fileSys);
+ cluster.restartNameNode();
+ fileSys = cluster.getFileSystem();
+ assertFalse(fileSys.getAclStatus(dir1).getEntries().isEmpty());
+ assertFalse(fileSys.getAclStatus(dir2).getEntries().isEmpty());
+ assertFalse(fileSys.getAclStatus(file1).getEntries().isEmpty());
+ assertTrue(fileSys.getAclStatus(dir3).getEntries().isEmpty());
+ assertTrue(fileSys.getAclStatus(file2).getEntries().isEmpty());
+ } finally {
+ IOUtils.cleanup(LOG, fileSys);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
[14/45] hadoop git commit: Fix CHANGES.txt to make it consistent with
branch-2.
Posted by vi...@apache.org.
Fix CHANGES.txt to make it consistent with branch-2.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e073b61d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e073b61d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e073b61d
Branch: refs/heads/HDFS-EC
Commit: e073b61dc42f0f999d409147458fcf38b8d578d7
Parents: 9f0319b
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Nov 12 20:02:28 2014 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Wed Nov 12 20:03:27 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 45 +++++++++++++++---------
1 file changed, 28 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e073b61d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7874b91..858a71a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -388,7 +388,7 @@ Release 2.7.0 - UNRELEASED
HDFS-7277. Remove explicit dependency on netty 3.2 in BKJournal. (wheat9)
- HDFS-7232. Populate hostname in httpfs audit log (Zoran Dimitrijevic
+ HDFS-7232. Populate hostname in httpfs audit log (Zoran Dimitrijevic
via aw)
HDFS-7258. CacheReplicationMonitor rescan schedule log should use DEBUG
@@ -398,6 +398,9 @@ Release 2.7.0 - UNRELEASED
TestBlockReaderFactory failures resulting from TemporarySocketDirectory GC.
(Jinghui Wang via Colin Patrick McCabe)
+ HDFS-7301. TestMissingBlocksAlert should use MXBeans instead of old web UI.
+ (Zhe Zhang via wheat9)
+
HDFS-7235. DataNode#transferBlock should report blocks that don't exist
using reportBadBlock (yzhang via cmccabe)
@@ -416,6 +419,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7389. Named user ACL cannot stop the user from accessing the FS entity.
(Vinayakumar B via cnauroth)
+ HDFS-6938. Cleanup javac warnings in FSNamesystem (Charles Lamb via wheat9)
+
Release 2.6.0 - 2014-11-15
INCOMPATIBLE CHANGES
@@ -601,6 +606,9 @@ Release 2.6.0 - 2014-11-15
HDFS-6959. Make the HDFS home directory location customizable. (yzhang via
cmccabe)
+ HDFS-6609. Use DirectorySnapshottableFeature to represent a snapshottable
+ directory. (Jing Zhao via wheat9)
+
HDFS-6886. Use single editlog record for creating file + overwrite. (Yi Liu
via jing9)
@@ -873,14 +881,9 @@ Release 2.6.0 - 2014-11-15
HDFS-6892. Add XDR packaging method for each NFS request (brandonli)
- HDFS-6938. Cleanup javac warnings in FSNamesystem (Charles Lamb via wheat9)
-
HDFS-6902. FileWriter should be closed in finally block in
BlockReceiver#receiveBlock() (Tsuyoshi OZAWA via Colin Patrick McCabe)
- HDFS-6800. Support Datanode layout changes with rolling upgrade.
- (James Thomas via Arpit Agarwal)
-
HDFS-6972. TestRefreshUserMappings.testRefreshSuperUserGroupsConfiguration
doesn't decode url correctly. (Yongjun Zhang via wang)
@@ -911,6 +914,11 @@ Release 2.6.0 - 2014-11-15
HDFS-6951. Correctly persist raw namespace xattrs to edit log and fsimage.
(clamb via wang)
+ HDFS-6800. Support Datanode layout changes with rolling upgrade.
+ (James Thomas via Arpit Agarwal)
+
+ HDFS-6981. Fix DN upgrade with layout version change. (Arpit Agarwal)
+
HDFS-6506. Newly moved block replica been invalidated and deleted in
TestBalancer. (Binglin Chang via cnauroth)
@@ -960,13 +968,16 @@ Release 2.6.0 - 2014-11-15
HDFS-7132. hdfs namenode -metadataVersion command does not honor
configured name dirs. (Charles Lamb via wang)
+ HDFS-7049. TestByteRangeInputStream.testPropagatedClose fails and throw
+ NPE on branch-2. (Juan Yu via wheat9)
+
HDFS-7148. TestEncryptionZones#testIsEncryptedMethod fails on branch-2
after archival storage merge. (wang)
HDFS-7157. Using Time.now() for recording start/end time of reconfiguration
tasks (Lei Xu via cmccabe)
- HDFS-6664. HDFS permissions guide documentation states incorrect default
+ HDFS-6664. HDFS permissions guide documentation states incorrect default
group mapping class. (Ray Chiang via aw)
HDFS-4227. Document dfs.namenode.resource.* (Daisuke Kobayashi via aw)
@@ -1025,7 +1036,16 @@ Release 2.6.0 - 2014-11-15
(clamb via yliu)
HADOOP-11233. hadoop.security.kms.client.encrypted.key.cache.expiry
- property spelled wrong in core-default. (Stephen Chu via yliu)
+ property spelled wrong in core-default. (Stephen Chu via yliu)
+
+ HDFS-7218. FSNamesystem ACL operations should write to audit log on
+ failure. (clamb via yliu)
+
+ HDFS-7199. DFSOutputStream should not silently drop data if DataStreamer
+ crashes with an unchecked exception (rushabhs via cmccabe)
+
+ HDFS-7383. DataNode.requestShortCircuitFdsForRead may throw
+ NullPointerException. (szetszwo via suresh)
HDFS-7147. Update archival storage user documentation.
(Tsz Wo Nicholas Sze via wheat9)
@@ -1039,12 +1059,6 @@ Release 2.6.0 - 2014-11-15
fails on Windows, because we cannot deny access to the file owner.
(Chris Nauroth via wheat9)
- HDFS-7218. FSNamesystem ACL operations should write to audit log on
- failure. (clamb via yliu)
-
- HDFS-7199. DFSOutputStream should not silently drop data if DataStreamer
- crashes with an unchecked exception (rushabhs via cmccabe)
-
HDFS-7367. HDFS short-circuit read cannot negotiate shared memory slot and
file descriptors when SASL is enabled on DataTransferProtocol.
(Chris Nauroth via wheat9)
@@ -1424,9 +1438,6 @@ Release 2.6.0 - 2014-11-15
HDFS-7382. DataNode in secure mode may throw NullPointerException if client
connects before DataNode registers itself with NameNode. (cnauroth)
- HDFS-7383. DataNode.requestShortCircuitFdsForRead may throw
- NullPointerException. (szetszwo via suresh)
-
HDFS-7387. NFS may only do partial commit due to a race between COMMIT and write
(brandonli)
[13/45] hadoop git commit: HDFS-7391. Renable SSLv2Hello in HttpFS.
Contributed by Robert Kanter.
Posted by vi...@apache.org.
HDFS-7391. Renable SSLv2Hello in HttpFS. Contributed by Robert Kanter.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f0319bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f0319bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f0319bb
Branch: refs/heads/HDFS-EC
Commit: 9f0319bba1788e4c579ce533b14c0deab63f28ee
Parents: 87818ef
Author: Arun C. Murthy <ac...@apache.org>
Authored: Wed Nov 12 18:51:46 2014 -0800
Committer: Arun C. Murthy <ac...@apache.org>
Committed: Wed Nov 12 18:51:46 2014 -0800
----------------------------------------------------------------------
.../hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f0319bb/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
index 614b702..c91c2e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
@@ -70,7 +70,7 @@
described in the APR documentation -->
<Connector port="${httpfs.http.port}" protocol="HTTP/1.1" SSLEnabled="true"
maxThreads="150" scheme="https" secure="true"
- clientAuth="false" sslEnabledProtocols="TLSv1"
+ clientAuth="false" sslEnabledProtocols="TLSv1,SSLv2Hello"
keystoreFile="${httpfs.ssl.keystore.file}"
keystorePass="${httpfs.ssl.keystore.pass}"/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f0319bb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cd8814e..7874b91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1430,6 +1430,8 @@ Release 2.6.0 - 2014-11-15
HDFS-7387. NFS may only do partial commit due to a race between COMMIT and write
(brandonli)
+ HDFS-7391. Renable SSLv2Hello in HttpFS. (rkanter via acmurthy)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
[45/45] hadoop git commit: Merge remote-tracking branch
'origin/trunk' into HDFS-EC
Posted by vi...@apache.org.
Merge remote-tracking branch 'origin/trunk' into HDFS-EC
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/041e97d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/041e97d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/041e97d7
Branch: refs/heads/HDFS-EC
Commit: 041e97d79c8fa835e8b08f0c92ef9e67029ed577
Parents: 95b3eba 26d3b7e
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Nov 17 11:58:47 2014 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Nov 17 11:58:47 2014 +0530
----------------------------------------------------------------------
dev-support/create-release.sh | 6 +
hadoop-common-project/hadoop-common/CHANGES.txt | 29 +-
.../hadoop-common/src/main/bin/hadoop | 2 +-
.../src/main/bin/hadoop-config.cmd | 16 +-
.../hadoop-common/src/main/bin/hadoop-config.sh | 6 +
.../src/main/bin/hadoop-daemons.sh | 19 +-
.../src/main/bin/hadoop-functions.sh | 22 +-
.../hadoop-common/src/main/bin/hadoop.cmd | 10 +-
.../hadoop-common/src/main/bin/slaves.sh | 2 +-
.../hadoop-common/src/main/bin/stop-all.sh | 2 +-
.../hadoop-common/src/main/conf/hadoop-env.sh | 3 +-
.../src/main/docs/releasenotes.html | 54 +-
.../org/apache/hadoop/fs/shell/Command.java | 28 +-
.../apache/hadoop/fs/shell/CommandFactory.java | 1 +
.../org/apache/hadoop/fs/shell/FsCommand.java | 2 +
.../org/apache/hadoop/fs/shell/find/And.java | 84 ++
.../hadoop/fs/shell/find/BaseExpression.java | 302 +++++++
.../apache/hadoop/fs/shell/find/Expression.java | 107 +++
.../hadoop/fs/shell/find/ExpressionFactory.java | 156 ++++
.../hadoop/fs/shell/find/FilterExpression.java | 144 +++
.../org/apache/hadoop/fs/shell/find/Find.java | 444 +++++++++
.../hadoop/fs/shell/find/FindOptions.java | 271 ++++++
.../org/apache/hadoop/fs/shell/find/Name.java | 100 +++
.../org/apache/hadoop/fs/shell/find/Print.java | 76 ++
.../org/apache/hadoop/fs/shell/find/Result.java | 88 ++
.../apache/hadoop/io/compress/CodecPool.java | 54 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 2 +-
.../apache/hadoop/security/SecurityUtil.java | 15 +
.../src/site/apt/CommandsManual.apt.vm | 7 +-
.../src/site/apt/FileSystemShell.apt.vm | 43 +
.../hadoop/fs/shell/find/MockFileSystem.java | 86 ++
.../apache/hadoop/fs/shell/find/TestAnd.java | 263 ++++++
.../fs/shell/find/TestFilterExpression.java | 145 +++
.../apache/hadoop/fs/shell/find/TestFind.java | 900 +++++++++++++++++++
.../apache/hadoop/fs/shell/find/TestHelper.java | 35 +
.../apache/hadoop/fs/shell/find/TestIname.java | 93 ++
.../apache/hadoop/fs/shell/find/TestName.java | 93 ++
.../apache/hadoop/fs/shell/find/TestPrint.java | 56 ++
.../apache/hadoop/fs/shell/find/TestPrint0.java | 56 ++
.../apache/hadoop/fs/shell/find/TestResult.java | 172 ++++
.../hadoop/io/compress/TestCodecPool.java | 47 +-
.../src/test/resources/testConf.xml | 44 +
.../hadoop-kms/src/main/tomcat/ssl-server.xml | 2 +-
.../src/main/tomcat/ssl-server.xml | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 68 +-
.../hadoop-hdfs/src/main/bin/hdfs | 3 +-
.../hadoop-hdfs/src/main/bin/hdfs.cmd | 10 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 61 +-
.../sasl/SaslDataTransferClient.java | 3 +-
.../sasl/SaslDataTransferServer.java | 3 +-
.../server/blockmanagement/BlockIdManager.java | 2 +-
.../server/datanode/SecureDataNodeStarter.java | 3 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 2 +
.../hdfs/server/namenode/FSEditLogOp.java | 12 +
.../hdfs/shortcircuit/ShortCircuitCache.java | 4 +-
.../hadoop/hdfs/util/ByteArrayManager.java | 15 +-
.../src/site/apt/HDFSCommands.apt.vm | 8 +-
.../java/org/apache/hadoop/hdfs/TestHFlush.java | 10 +-
.../hdfs/server/namenode/TestEditLog.java | 67 ++
.../hadoop/hdfs/util/TestByteArrayManager.java | 43 +-
.../src/test/resources/testHDFSConf.xml | 223 +++++
hadoop-mapreduce-project/CHANGES.txt | 11 +-
hadoop-mapreduce-project/bin/mapred | 13 +-
hadoop-mapreduce-project/bin/mapred-config.sh | 2 +-
hadoop-mapreduce-project/bin/mapred.cmd | 11 +-
hadoop-mapreduce-project/conf/mapred-env.cmd | 2 +-
.../apache/hadoop/mapred/LineRecordReader.java | 1 +
.../mapreduce/lib/input/LineRecordReader.java | 1 +
.../hadoop/mapreduce/task/reduce/Fetcher.java | 39 +-
.../src/site/apt/MapredCommands.apt.vm | 5 +-
.../hadoop/mapred/TestLineRecordReader.java | 37 +
.../lib/input/TestLineRecordReader.java | 38 +
hadoop-project/src/site/site.xml | 1 +
hadoop-yarn-project/CHANGES.txt | 54 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 +-
.../hadoop-yarn/bin/yarn-config.sh | 2 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 10 +-
.../hadoop-yarn/conf/yarn-env.cmd | 2 +-
.../hadoop/yarn/api/records/LocalResource.java | 38 +
.../hadoop/yarn/conf/YarnConfiguration.java | 33 +-
.../src/main/proto/yarn_protos.proto | 1 +
.../hadoop/yarn/ContainerLogAppender.java | 7 +-
.../records/impl/pb/LocalResourcePBImpl.java | 20 +
.../yarn/sharedcache/ChecksumSHA256Impl.java | 37 +
.../yarn/sharedcache/SharedCacheChecksum.java | 43 +
.../sharedcache/SharedCacheChecksumFactory.java | 84 ++
.../org/apache/hadoop/yarn/util/FSDownload.java | 5 +-
.../src/main/resources/yarn-default.xml | 30 +-
.../hadoop/yarn/TestContainerLogAppender.java | 46 +
...pplicationHistoryManagerOnTimelineStore.java | 7 +-
.../TestApplicationHistoryClientService.java | 4 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 10 +-
.../server/nodemanager/ContainerExecutor.java | 52 +-
.../nodemanager/DockerContainerExecutor.java | 794 ++++++++++++++++
.../nodemanager/LinuxContainerExecutor.java | 2 +-
.../yarn/server/nodemanager/NodeManager.java | 2 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 5 +-
.../containermanager/ContainerManagerImpl.java | 13 +
.../container/ContainerImpl.java | 72 +-
.../launcher/ContainerLaunch.java | 36 +-
.../launcher/RecoveredContainerLaunch.java | 20 +-
.../localizer/LocalResourceRequest.java | 11 +
.../localizer/ResourceLocalizationService.java | 6 +-
.../sharedcache/SharedCacheUploadEvent.java | 58 ++
.../sharedcache/SharedCacheUploadEventType.java | 28 +
.../sharedcache/SharedCacheUploadService.java | 126 +++
.../sharedcache/SharedCacheUploader.java | 289 ++++++
.../logaggregation/AppLogAggregatorImpl.java | 7 +-
.../recovery/NMLeveldbStateStoreService.java | 24 +-
.../TestDockerContainerExecutor.java | 213 +++++
.../TestDockerContainerExecutorWithMocks.java | 259 ++++++
.../container/TestContainer.java | 2 +-
.../launcher/TestContainerLaunch.java | 10 +-
.../TestResourceLocalizationService.java | 2 +-
.../TestSharedCacheUploadService.java | 50 ++
.../sharedcache/TestSharedCacheUploader.java | 241 +++++
.../TestNMLeveldbStateStoreService.java | 7 +
.../ApplicationMasterService.java | 1 +
.../server/resourcemanager/RMAppManager.java | 9 +-
.../resourcemanager/recovery/RMStateStore.java | 2 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 27 +-
.../scheduler/fair/FairScheduler.java | 18 +-
.../server/resourcemanager/TestAppManager.java | 12 +-
.../yarn/server/resourcemanager/TestRM.java | 111 +++
.../rmapp/TestRMAppTransitions.java | 29 +-
.../scheduler/fair/TestFairScheduler.java | 88 +-
.../server/webproxy/WebAppProxyServlet.java | 1 +
.../src/site/apt/DockerContainerExecutor.apt.vm | 200 +++++
.../src/site/apt/YarnCommands.apt.vm | 5 +-
129 files changed, 7782 insertions(+), 273 deletions(-)
----------------------------------------------------------------------
[36/45] hadoop git commit: Revert HADOOP-11257 because it didn't get
a jenkins run yet
Posted by vi...@apache.org.
Revert HADOOP-11257 because it didn't get a jenkins run yet
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f95017b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f95017b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f95017b3
Branch: refs/heads/HDFS-EC
Commit: f95017b3ead4291e08ce9b6976409efe78995b6e
Parents: 10c98ae
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Fri Nov 14 13:06:03 2014 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Fri Nov 14 13:06:03 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/src/main/bin/hadoop | 5 -----
hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd | 7 -------
.../hadoop-common/src/site/apt/CommandsManual.apt.vm | 3 ---
3 files changed, 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f95017b3/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 9df2c7d..eccd322 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -33,8 +33,6 @@ function hadoop_usage()
echo " copy file or directories recursively"
echo " fs run a generic filesystem user client"
echo " jar <jar> run a jar file"
- echo " note: please use \"yarn jar\" to launch"
- echo " YARN applications, not this command."
echo " jnipath prints the java.library.path"
echo " key manage keys via the KeyProvider"
echo " trace view and modify Hadoop tracing settings"
@@ -152,9 +150,6 @@ case ${COMMAND} in
CLASS=org.apache.hadoop.fs.FsShell
;;
jar)
- if [[ -n "${YARN_OPTS}" ]] || [[ -n "${YARN_CLIENT_OPTS}" ]]; then
- hadoop_error "WARNING: Use \"yarn jar\" to launch YARN applications."
- fi
CLASS=org.apache.hadoop.util.RunJar
;;
jnipath)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f95017b3/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
index c537539..fc2e9b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
@@ -177,11 +177,6 @@ call :updatepath %HADOOP_BIN_PATH%
goto :eof
:version
- if defined YARN_OPTS (
- @echo WARNING: Use "yarn jar" to launch YARN applications.
- ) else if defined YARN_CLIENT_OPTS (
- @echo WARNING: Use "yarn jar" to launch YARN applications.
- )
set CLASS=org.apache.hadoop.util.VersionInfo
goto :eof
@@ -266,8 +261,6 @@ call :updatepath %HADOOP_BIN_PATH%
goto :MakeCmdArgsLoop
:EndLoop
set hadoop-command-arguments=%_arguments%
- @echo note: please use "yarn jar" to launch
- @echo YARN applications, not this command.
goto :eof
:print_usage
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f95017b3/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
index 6d2fd5e..7771aa1 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
@@ -177,9 +177,6 @@ User Commands
Word count example is also run using jar command. It can be referred from
Wordcount example
- Use {{{../../hadoop-yarn/hadoop-yarn-site/YarnCommands.html#jar}<<<yarn jar>>>}}
- to launch YARN applications instead.
-
* <<<job>>>
Deprecated. Use
[07/45] hadoop git commit: YARN-2794. Fixed log messages about
distributing system-credentials. Contributed by Jian He.
Posted by vi...@apache.org.
YARN-2794. Fixed log messages about distributing system-credentials. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be7bf956
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be7bf956
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be7bf956
Branch: refs/heads/HDFS-EC
Commit: be7bf956e96dd0fd9b521ca71df9124b9cc5ebd0
Parents: 51957ca
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Nov 12 11:07:57 2014 -0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Wed Nov 12 11:07:57 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/server/nodemanager/NodeManager.java | 2 +-
.../hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 5 ++++-
.../localizer/ResourceLocalizationService.java | 6 +++---
.../containermanager/logaggregation/AppLogAggregatorImpl.java | 7 ++++---
5 files changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be7bf956/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f869fde..1f991f6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -949,6 +949,9 @@ Release 2.6.0 - 2014-11-15
YARN-2843. Fixed NodeLabelsManager to trim inputs for hosts and labels so
as to make them work correctly. (Wangda Tan via vinodkv)
+ YARN-2794. Fixed log messages about distributing system-credentials. (Jian He via
+ zjshen)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be7bf956/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 4f90bf5..53cbb11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -317,7 +317,7 @@ public class NodeManager extends CompositeService
protected final ConcurrentMap<ApplicationId, Application> applications =
new ConcurrentHashMap<ApplicationId, Application>();
- private Map<ApplicationId, Credentials> systemCredentials =
+ private volatile Map<ApplicationId, Credentials> systemCredentials =
new HashMap<ApplicationId, Credentials>();
protected final ConcurrentMap<ContainerId, Container> containers =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be7bf956/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index fcdd2c1..f561dbb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -552,7 +552,10 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
map.put(entry.getKey(), credentials);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Retrieved credentials form RM: " + map);
+ for (Map.Entry<ApplicationId, Credentials> entry : map.entrySet()) {
+ LOG.debug("Retrieved credentials form RM for " + entry.getKey() + ": "
+ + entry.getValue().getAllTokens());
+ }
}
return map;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be7bf956/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index cb56d67..f4b6221 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1122,9 +1122,9 @@ public class ResourceLocalizationService extends CompositeService
if (systemCredentials == null) {
return null;
}
- for (Token<?> token : systemCredentials.getAllTokens()) {
- LOG.info("Adding new framework-token for " + appId
- + " for localization: " + token);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding new framework-token for " + appId
+ + " for localization: " + systemCredentials.getAllTokens());
}
return systemCredentials;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be7bf956/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 3174c5c..20887b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -202,9 +202,10 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
Credentials systemCredentials =
context.getSystemCredentialsForApps().get(appId);
if (systemCredentials != null) {
- for (Token<?> token : systemCredentials.getAllTokens()) {
- LOG.info("Adding new framework-token for " + appId
- + " for log-aggregation: " + token + " user=" + userUgi);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding new framework-token for " + appId
+ + " for log-aggregation: " + systemCredentials.getAllTokens()
+ + "; userUgi=" + userUgi);
}
// this will replace old token
userUgi.addCredentials(systemCredentials);
[23/45] hadoop git commit: HADOOP-8989. hadoop fs -find feature
(Jonathan Allen via aw)
Posted by vi...@apache.org.
HADOOP-8989. hadoop fs -find feature (Jonathan Allen via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba879a5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba879a5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba879a5d
Branch: refs/heads/HDFS-EC
Commit: ba879a5dadbb0f33bba7e05ebc329a9942f34276
Parents: 81dc0ac
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Nov 13 08:20:43 2014 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Nov 13 11:52:38 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../org/apache/hadoop/fs/shell/Command.java | 28 +-
.../apache/hadoop/fs/shell/CommandFactory.java | 1 +
.../org/apache/hadoop/fs/shell/FsCommand.java | 2 +
.../org/apache/hadoop/fs/shell/find/And.java | 84 ++
.../hadoop/fs/shell/find/BaseExpression.java | 302 +++++++
.../apache/hadoop/fs/shell/find/Expression.java | 107 +++
.../hadoop/fs/shell/find/ExpressionFactory.java | 156 ++++
.../hadoop/fs/shell/find/FilterExpression.java | 144 +++
.../org/apache/hadoop/fs/shell/find/Find.java | 444 +++++++++
.../hadoop/fs/shell/find/FindOptions.java | 271 ++++++
.../org/apache/hadoop/fs/shell/find/Name.java | 100 +++
.../org/apache/hadoop/fs/shell/find/Print.java | 76 ++
.../org/apache/hadoop/fs/shell/find/Result.java | 88 ++
.../src/site/apt/FileSystemShell.apt.vm | 43 +
.../hadoop/fs/shell/find/MockFileSystem.java | 86 ++
.../apache/hadoop/fs/shell/find/TestAnd.java | 263 ++++++
.../fs/shell/find/TestFilterExpression.java | 145 +++
.../apache/hadoop/fs/shell/find/TestFind.java | 900 +++++++++++++++++++
.../apache/hadoop/fs/shell/find/TestHelper.java | 35 +
.../apache/hadoop/fs/shell/find/TestIname.java | 93 ++
.../apache/hadoop/fs/shell/find/TestName.java | 93 ++
.../apache/hadoop/fs/shell/find/TestPrint.java | 56 ++
.../apache/hadoop/fs/shell/find/TestPrint0.java | 56 ++
.../apache/hadoop/fs/shell/find/TestResult.java | 172 ++++
.../src/test/resources/testConf.xml | 44 +
.../src/test/resources/testHDFSConf.xml | 223 +++++
27 files changed, 4013 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 732cdc7..d77d024 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -360,6 +360,8 @@ Release 2.7.0 - UNRELEASED
HADOOP-7984. Add hadoop --loglevel option to change log level.
(Akira AJISAKA via cnauroth)
+ HADOOP-8989. hadoop fs -find feature (Jonathan Allen via aw)
+
IMPROVEMENTS
HADOOP-11156. DelegateToFileSystem should implement
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
index 8c5e880..c573aa0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java
@@ -65,6 +65,8 @@ abstract public class Command extends Configured {
public PrintStream out = System.out;
/** allows stderr to be captured if necessary */
public PrintStream err = System.err;
+ /** allows the command factory to be used if necessary */
+ private CommandFactory commandFactory = null;
/** Constructor */
protected Command() {
@@ -121,6 +123,15 @@ abstract public class Command extends Configured {
return exitCode;
}
+ /** sets the command factory for later use */
+ public void setCommandFactory(CommandFactory factory) {
+ this.commandFactory = factory;
+ }
+ /** retrieves the command factory */
+ protected CommandFactory getCommandFactory() {
+ return this.commandFactory;
+ }
+
/**
* Invokes the command handler. The default behavior is to process options,
* expand arguments, and then process each argument.
@@ -308,7 +319,7 @@ abstract public class Command extends Configured {
for (PathData item : items) {
try {
processPath(item);
- if (recursive && item.stat.isDirectory()) {
+ if (recursive && isPathRecursable(item)) {
recursePath(item);
}
postProcessPath(item);
@@ -319,6 +330,21 @@ abstract public class Command extends Configured {
}
/**
+ * Determines whether a {@link PathData} item is recursable. Default
+ * implementation is to recurse directories but can be overridden to recurse
+ * through symbolic links.
+ *
+ * @param item
+ * a {@link PathData} object
+ * @return true if the item is recursable, false otherwise
+ * @throws IOException
+ * if anything goes wrong in the user-implementation
+ */
+ protected boolean isPathRecursable(PathData item) throws IOException {
+ return item.stat.isDirectory();
+ }
+
+ /**
* Hook for commands to implement an operation to be applied on each
* path for the command. Note implementation of this method is optional
* if earlier methods in the chain handle the operation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFactory.java
index dec8373..9b128cf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandFactory.java
@@ -124,6 +124,7 @@ public class CommandFactory extends Configured {
if (cmdClass != null) {
instance = ReflectionUtils.newInstance(cmdClass, conf);
instance.setName(cmdName);
+ instance.setCommandFactory(this);
}
}
return instance;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
index 3372809..cc8fbb4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FsShellPermissions;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.find.Find;
/**
* Base class for all "hadoop fs" commands
@@ -48,6 +49,7 @@ abstract public class FsCommand extends Command {
factory.registerCommands(Count.class);
factory.registerCommands(Delete.class);
factory.registerCommands(Display.class);
+ factory.registerCommands(Find.class);
factory.registerCommands(FsShellPermissions.class);
factory.registerCommands(FsUsage.class);
factory.registerCommands(Ls.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/And.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/And.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/And.java
new file mode 100644
index 0000000..ced489c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/And.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.io.IOException;
+import java.util.Deque;
+
+import org.apache.hadoop.fs.shell.PathData;
+
+/**
+ * Implements the -a (and) operator for the
+ * {@link org.apache.hadoop.fs.shell.find.Find} command.
+ */
+final class And extends BaseExpression {
+ /** Registers this expression with the specified factory. */
+ public static void registerExpression(ExpressionFactory factory)
+ throws IOException {
+ factory.addClass(And.class, "-a");
+ factory.addClass(And.class, "-and");
+ }
+
+ private static final String[] USAGE = { "expression -a expression",
+ "expression -and expression", "expression expression" };
+ private static final String[] HELP = {
+ "Logical AND operator for joining two expressions. Returns",
+ "true if both child expressions return true. Implied by the",
+ "juxtaposition of two expressions and so does not need to be",
+ "explicitly specified. The second expression will not be",
+ "applied if the first fails." };
+
+ public And() {
+ super();
+ setUsage(USAGE);
+ setHelp(HELP);
+ }
+
+ /**
+ * Applies child expressions to the {@link PathData} item. If all pass then
+ * returns {@link Result#PASS} else returns the result of the first
+ * non-passing expression.
+ */
+ @Override
+ public Result apply(PathData item, int depth) throws IOException {
+ Result result = Result.PASS;
+ for (Expression child : getChildren()) {
+ Result childResult = child.apply(item, -1);
+ result = result.combine(childResult);
+ if (!result.isPass()) {
+ return result;
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public boolean isOperator() {
+ return true;
+ }
+
+ @Override
+ public int getPrecedence() {
+ return 200;
+ }
+
+ @Override
+ public void addChildren(Deque<Expression> expressions) {
+ addChildren(expressions, 2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java
new file mode 100644
index 0000000..db7d62f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/BaseExpression.java
@@ -0,0 +1,302 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.io.IOException;
+import java.util.Deque;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.PathData;
+
+/**
+ * Abstract expression for use in the
+ * {@link org.apache.hadoop.fs.shell.find.Find} command. Provides default
+ * behavior for a no-argument primary expression.
+ */
+public abstract class BaseExpression implements Expression, Configurable {
+ private String[] usage = { "Not yet implemented" };
+ private String[] help = { "Not yet implemented" };
+
+ /** Sets the usage text for this {@link Expression} */
+ protected void setUsage(String[] usage) {
+ this.usage = usage;
+ }
+
+ /** Sets the help text for this {@link Expression} */
+ protected void setHelp(String[] help) {
+ this.help = help;
+ }
+
+ @Override
+ public String[] getUsage() {
+ return this.usage;
+ }
+
+ @Override
+ public String[] getHelp() {
+ return this.help;
+ }
+
+ @Override
+ public void setOptions(FindOptions options) throws IOException {
+ this.options = options;
+ for (Expression child : getChildren()) {
+ child.setOptions(options);
+ }
+ }
+
+ @Override
+ public void prepare() throws IOException {
+ for (Expression child : getChildren()) {
+ child.prepare();
+ }
+ }
+
+ @Override
+ public void finish() throws IOException {
+ for (Expression child : getChildren()) {
+ child.finish();
+ }
+ }
+
+ /** Options passed in from the {@link Find} command. */
+ private FindOptions options;
+
+ /** Hadoop configuration. */
+ private Configuration conf;
+
+ /** Arguments for this expression. */
+ private LinkedList<String> arguments = new LinkedList<String>();
+
+ /** Children of this expression. */
+ private LinkedList<Expression> children = new LinkedList<Expression>();
+
+ /** Return the options to be used by this expression. */
+ protected FindOptions getOptions() {
+ return (this.options == null) ? new FindOptions() : this.options;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getClass().getSimpleName());
+ sb.append("(");
+ boolean firstArg = true;
+ for (String arg : getArguments()) {
+ if (!firstArg) {
+ sb.append(",");
+ } else {
+ firstArg = false;
+ }
+ sb.append(arg);
+ }
+ sb.append(";");
+ firstArg = true;
+ for (Expression child : getChildren()) {
+ if (!firstArg) {
+ sb.append(",");
+ } else {
+ firstArg = false;
+ }
+ sb.append(child.toString());
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+
+ @Override
+ public boolean isAction() {
+ for (Expression child : getChildren()) {
+ if (child.isAction()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean isOperator() {
+ return false;
+ }
+
+ /**
+ * Returns the arguments of this expression
+ *
+ * @return list of argument strings
+ */
+ protected List<String> getArguments() {
+ return this.arguments;
+ }
+
+ /**
+ * Returns the argument at the given position (starting from 1).
+ *
+ * @param position
+ * argument to be returned
+ * @return requested argument
+ * @throws IOException
+ * if the argument doesn't exist or is null
+ */
+ protected String getArgument(int position) throws IOException {
+ if (position > this.arguments.size()) {
+ throw new IOException("Missing argument at " + position);
+ }
+ String argument = this.arguments.get(position - 1);
+ if (argument == null) {
+ throw new IOException("Null argument at position " + position);
+ }
+ return argument;
+ }
+
+ /**
+ * Returns the children of this expression.
+ *
+ * @return list of child expressions
+ */
+ protected List<Expression> getChildren() {
+ return this.children;
+ }
+
+ @Override
+ public int getPrecedence() {
+ return 0;
+ }
+
+ @Override
+ public void addChildren(Deque<Expression> exprs) {
+ // no children by default, will be overridden by specific expressions.
+ }
+
+ /**
+ * Add a specific number of children to this expression. The children are
+ * popped off the head of the expressions.
+ *
+ * @param exprs
+ * deque of expressions from which to take the children
+ * @param count
+ * number of children to be added
+ */
+ protected void addChildren(Deque<Expression> exprs, int count) {
+ for (int i = 0; i < count; i++) {
+ addChild(exprs.pop());
+ }
+ }
+
+ /**
+ * Add a single argument to this expression. The argument is popped off the
+ * head of the expressions.
+ *
+ * @param expr
+ * child to add to the expression
+ */
+ private void addChild(Expression expr) {
+ children.push(expr);
+ }
+
+ @Override
+ public void addArguments(Deque<String> args) {
+ // no children by default, will be overridden by specific expressions.
+ }
+
+ /**
+ * Add a specific number of arguments to this expression. The children are
+ * popped off the head of the expressions.
+ *
+ * @param args
+ * deque of arguments from which to take the argument
+ * @param count
+ * number of children to be added
+ */
+ protected void addArguments(Deque<String> args, int count) {
+ for (int i = 0; i < count; i++) {
+ addArgument(args.pop());
+ }
+ }
+
+ /**
+ * Add a single argument to this expression. The argument is popped off the
+ * head of the expressions.
+ *
+ * @param arg
+ * argument to add to the expression
+ */
+ protected void addArgument(String arg) {
+ arguments.add(arg);
+ }
+
+ /**
+ * Returns the {@link FileStatus} from the {@link PathData} item. If the
+ * current options require links to be followed then the returned file status
+ * is that of the linked file.
+ *
+ * @param item
+ * PathData
+ * @param depth
+ * current depth in the process directories
+ * @return FileStatus
+ */
+ protected FileStatus getFileStatus(PathData item, int depth)
+ throws IOException {
+ FileStatus fileStatus = item.stat;
+ if (fileStatus.isSymlink()) {
+ if (options.isFollowLink() || (options.isFollowArgLink() &&
+ (depth == 0))) {
+ Path linkedFile = item.fs.resolvePath(fileStatus.getSymlink());
+ fileStatus = getFileSystem(item).getFileStatus(linkedFile);
+ }
+ }
+ return fileStatus;
+ }
+
+ /**
+ * Returns the {@link Path} from the {@link PathData} item.
+ *
+ * @param item
+ * PathData
+ * @return Path
+ */
+ protected Path getPath(PathData item) throws IOException {
+ return item.path;
+ }
+
+ /**
+ * Returns the {@link FileSystem} associated with the {@link PathData} item.
+ *
+ * @param item PathData
+ * @return FileSystem
+ */
+ protected FileSystem getFileSystem(PathData item) throws IOException {
+ return item.fs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java
new file mode 100644
index 0000000..ccad631
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Expression.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.io.IOException;
+import java.util.Deque;
+
+import org.apache.hadoop.fs.shell.PathData;
+
+/**
+ * Interface describing an expression to be used in the
+ * {@link org.apache.hadoop.fs.shell.find.Find} command.
+ */
+public interface Expression {
+ /**
+ * Set the options for this expression, called once before processing any
+ * items.
+ */
+ public void setOptions(FindOptions options) throws IOException;
+
+ /**
+ * Prepares the expression for execution, called once after setting options
+ * and before processing any options.
+ * @throws IOException
+ */
+ public void prepare() throws IOException;
+
+ /**
+ * Apply the expression to the specified item, called once for each item.
+ *
+ * @param item {@link PathData} item to be processed
+ * @param depth distance of the item from the command line argument
+ * @return {@link Result} of applying the expression to the item
+ */
+ public Result apply(PathData item, int depth) throws IOException;
+
+ /**
+ * Finishes the expression, called once after processing all items.
+ *
+ * @throws IOException
+ */
+ public void finish() throws IOException;
+
+ /**
+ * Returns brief usage instructions for this expression. Multiple items should
+ * be returned if there are multiple ways to use this expression.
+ *
+ * @return array of usage instructions
+ */
+ public String[] getUsage();
+
+ /**
+ * Returns a description of the expression for use in help. Multiple lines
+ * should be returned array items. Lines should be formated to 60 characters
+ * or less.
+ *
+ * @return array of description lines
+ */
+ public String[] getHelp();
+
+ /**
+ * Indicates whether this expression performs an action, i.e. provides output
+ * back to the user.
+ */
+ public boolean isAction();
+
+ /** Identifies the expression as an operator rather than a primary. */
+ public boolean isOperator();
+
+ /**
+ * Returns the precedence of this expression
+ * (only applicable to operators).
+ */
+ public int getPrecedence();
+
+ /**
+ * Adds children to this expression. Children are popped from the head of the
+ * deque.
+ *
+ * @param expressions
+ * deque of expressions from which to take the children
+ */
+ public void addChildren(Deque<Expression> expressions);
+
+ /**
+ * Adds arguments to this expression. Arguments are popped from the head of
+ * the deque and added to the front of the child list, ie last child added is
+ * the first evaluated.
+ * @param args deque of arguments from which to take expression arguments
+ */
+ public void addArguments(Deque<String> args);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/ExpressionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/ExpressionFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/ExpressionFactory.java
new file mode 100644
index 0000000..b922a9e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/ExpressionFactory.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Factory class for registering and searching for expressions for use in the
+ * {@link org.apache.hadoop.fs.shell.find.Find} command.
+ */
+final class ExpressionFactory {
+ private static final String REGISTER_EXPRESSION_METHOD = "registerExpression";
+ private Map<String, Class<? extends Expression>> expressionMap =
+ new HashMap<String, Class<? extends Expression>>();
+
+ private static final ExpressionFactory INSTANCE = new ExpressionFactory();
+
+ static ExpressionFactory getExpressionFactory() {
+ return INSTANCE;
+ }
+
+ /**
+ * Private constructor to ensure singleton.
+ */
+ private ExpressionFactory() {
+ }
+
+ /**
+ * Invokes "static void registerExpression(FindExpressionFactory)" on the
+ * given class. This method abstracts the contract between the factory and the
+ * expression class. Do not assume that directly invoking registerExpression
+ * on the given class will have the same effect.
+ *
+ * @param expressionClass
+ * class to allow an opportunity to register
+ */
+ void registerExpression(Class<? extends Expression> expressionClass) {
+ try {
+ Method register = expressionClass.getMethod(REGISTER_EXPRESSION_METHOD,
+ ExpressionFactory.class);
+ if (register != null) {
+ register.invoke(null, this);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(StringUtils.stringifyException(e));
+ }
+ }
+
+ /**
+ * Register the given class as handling the given list of expression names.
+ *
+ * @param expressionClass
+ * the class implementing the expression names
+ * @param names
+ * one or more command names that will invoke this class
+ * @throws IOException
+ * if the expression is not of an expected type
+ */
+ void addClass(Class<? extends Expression> expressionClass,
+ String... names) throws IOException {
+ for (String name : names)
+ expressionMap.put(name, expressionClass);
+ }
+
+ /**
+ * Determines whether the given expression name represents and actual
+ * expression.
+ *
+ * @param expressionName
+ * name of the expression
+ * @return true if expressionName represents an expression
+ */
+ boolean isExpression(String expressionName) {
+ return expressionMap.containsKey(expressionName);
+ }
+
+ /**
+ * Get an instance of the requested expression
+ *
+ * @param expressionName
+ * name of the command to lookup
+ * @param conf
+ * the Hadoop configuration
+ * @return the {@link Expression} or null if the expression is unknown
+ */
+ Expression getExpression(String expressionName, Configuration conf) {
+ if (conf == null)
+ throw new NullPointerException("configuration is null");
+
+ Class<? extends Expression> expressionClass = expressionMap
+ .get(expressionName);
+ Expression instance = createExpression(expressionClass, conf);
+ return instance;
+ }
+
+ /**
+ * Creates an instance of the requested {@link Expression} class.
+ *
+ * @param expressionClass
+ * {@link Expression} class to be instantiated
+ * @param conf
+ * the Hadoop configuration
+ * @return a new instance of the requested {@link Expression} class
+ */
+ Expression createExpression(
+ Class<? extends Expression> expressionClass, Configuration conf) {
+ Expression instance = null;
+ if (expressionClass != null) {
+ instance = ReflectionUtils.newInstance(expressionClass, conf);
+ }
+ return instance;
+ }
+
+ /**
+ * Creates an instance of the requested {@link Expression} class.
+ *
+ * @param expressionClassname
+ * name of the {@link Expression} class to be instantiated
+ * @param conf
+ * the Hadoop configuration
+ * @return a new instance of the requested {@link Expression} class
+ */
+ Expression createExpression(String expressionClassname,
+ Configuration conf) {
+ try {
+ Class<? extends Expression> expressionClass = Class.forName(
+ expressionClassname).asSubclass(Expression.class);
+ return createExpression(expressionClass, conf);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException("Invalid classname "
+ + expressionClassname);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FilterExpression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FilterExpression.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FilterExpression.java
new file mode 100644
index 0000000..0ebb0fa
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FilterExpression.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.io.IOException;
+import java.util.Deque;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.shell.PathData;
+
+/**
+ * Provides an abstract composition filter for the {@link Expression} interface.
+ * Allows other {@link Expression} implementations to be reused without
+ * inheritance.
+ */
+public abstract class FilterExpression implements Expression, Configurable {
+ protected Expression expression;
+
+ protected FilterExpression(Expression expression) {
+ this.expression = expression;
+ }
+
+ @Override
+ public void setOptions(FindOptions options) throws IOException {
+ if (expression != null) {
+ expression.setOptions(options);
+ }
+ }
+
+ @Override
+ public void prepare() throws IOException {
+ if (expression != null) {
+ expression.prepare();
+ }
+ }
+
+ @Override
+ public Result apply(PathData item, int depth) throws IOException {
+ if (expression != null) {
+ return expression.apply(item, -1);
+ }
+ return Result.PASS;
+ }
+
+ @Override
+ public void finish() throws IOException {
+ if (expression != null) {
+ expression.finish();
+ }
+ }
+
+ @Override
+ public String[] getUsage() {
+ if (expression != null) {
+ return expression.getUsage();
+ }
+ return null;
+ }
+
+ @Override
+ public String[] getHelp() {
+ if (expression != null) {
+ return expression.getHelp();
+ }
+ return null;
+ }
+
+ @Override
+ public boolean isAction() {
+ if (expression != null) {
+ return expression.isAction();
+ }
+ return false;
+ }
+
+ @Override
+ public boolean isOperator() {
+ if (expression != null) {
+ return expression.isOperator();
+ }
+ return false;
+ }
+
+ @Override
+ public int getPrecedence() {
+ if (expression != null) {
+ return expression.getPrecedence();
+ }
+ return -1;
+ }
+
+ @Override
+ public void addChildren(Deque<Expression> expressions) {
+ if (expression != null) {
+ expression.addChildren(expressions);
+ }
+ }
+
+ @Override
+ public void addArguments(Deque<String> args) {
+ if (expression != null) {
+ expression.addArguments(args);
+ }
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ if (expression instanceof Configurable) {
+ ((Configurable) expression).setConf(conf);
+ }
+ }
+
+ @Override
+ public Configuration getConf() {
+ if (expression instanceof Configurable) {
+ return ((Configurable) expression).getConf();
+ }
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ if (expression != null) {
+ return getClass().getSimpleName() + "-" + expression.toString();
+ }
+ return getClass().getSimpleName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Find.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Find.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Find.java
new file mode 100644
index 0000000..05cd818
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Find.java
@@ -0,0 +1,444 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.CommandFactory;
+import org.apache.hadoop.fs.shell.CommandFormat;
+import org.apache.hadoop.fs.shell.FsCommand;
+import org.apache.hadoop.fs.shell.PathData;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+/**
+ * Implements a Hadoop find command.
+ */
+public class Find extends FsCommand {
+ /**
+ * Register the names for the count command
+ *
+ * @param factory the command factory that will instantiate this class
+ */
+ public static void registerCommands(CommandFactory factory) {
+ factory.addClass(Find.class, "-find");
+ }
+
+ public static final String NAME = "find";
+ public static final String USAGE = "<path> ... <expression> ...";
+ public static final String DESCRIPTION;
+ private static String[] HELP =
+ { "Finds all files that match the specified expression and",
+ "applies selected actions to them. If no <path> is specified",
+ "then defaults to the current working directory. If no",
+ "expression is specified then defaults to -print."
+ };
+
+ private static final String OPTION_FOLLOW_LINK = "L";
+ private static final String OPTION_FOLLOW_ARG_LINK = "H";
+
+ /** List of expressions recognized by this command. */
+ @SuppressWarnings("rawtypes")
+ private static final Class[] EXPRESSIONS;
+
+ static {
+ // Initialize the static variables.
+ EXPRESSIONS = new Class[] {
+ // Operator Expressions
+ And.class,
+ // Action Expressions
+ Print.class,
+ // Navigation Expressions
+ // Matcher Expressions
+ Name.class };
+ DESCRIPTION = buildDescription(ExpressionFactory.getExpressionFactory());
+
+ // Register the expressions with the expression factory.
+ registerExpressions(ExpressionFactory.getExpressionFactory());
+ }
+
+ /** Options for use in this command */
+ private FindOptions options;
+
+ /** Root expression for this instance of the command. */
+ private Expression rootExpression;
+
+ /** Set of path items returning a {@link Result#STOP} result. */
+ private HashSet<Path> stopPaths = new HashSet<Path>();
+
+ /** Register the expressions with the expression factory. */
+ @SuppressWarnings("unchecked")
+ private static void registerExpressions(ExpressionFactory factory) {
+ for (Class<? extends Expression> exprClass : EXPRESSIONS) {
+ factory.registerExpression(exprClass);
+ }
+ }
+
+ /** Build the description used by the help command. */
+ @SuppressWarnings("unchecked")
+ private static String buildDescription(ExpressionFactory factory) {
+ ArrayList<Expression> operators = new ArrayList<Expression>();
+ ArrayList<Expression> primaries = new ArrayList<Expression>();
+ for (Class<? extends Expression> exprClass : EXPRESSIONS) {
+ Expression expr = factory.createExpression(exprClass, null);
+ if (expr.isOperator()) {
+ operators.add(expr);
+ } else {
+ primaries.add(expr);
+ }
+ }
+ Collections.sort(operators, new Comparator<Expression>() {
+ @Override
+ public int compare(Expression arg0, Expression arg1) {
+ return arg0.getClass().getName().compareTo(arg1.getClass().getName());
+ }
+ });
+ Collections.sort(primaries, new Comparator<Expression>() {
+ @Override
+ public int compare(Expression arg0, Expression arg1) {
+ return arg0.getClass().getName().compareTo(arg1.getClass().getName());
+ }
+ });
+
+ StringBuilder sb = new StringBuilder();
+ for (String line : HELP) {
+ sb.append(line).append("\n");
+ }
+ sb.append("\n");
+ sb.append("The following primary expressions are recognised:\n");
+ for (Expression expr : primaries) {
+ for (String line : expr.getUsage()) {
+ sb.append(" ").append(line).append("\n");
+ }
+ for (String line : expr.getHelp()) {
+ sb.append(" ").append(line).append("\n");
+ }
+ sb.append("\n");
+ }
+ sb.append("The following operators are recognised:\n");
+ for (Expression expr : operators) {
+ for (String line : expr.getUsage()) {
+ sb.append(" ").append(line).append("\n");
+ }
+ for (String line : expr.getHelp()) {
+ sb.append(" ").append(line).append("\n");
+ }
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+
+ /** Default constructor for the Find command. */
+ public Find() {
+ setRecursive(true);
+ }
+
+ @Override
+ protected void processOptions(LinkedList<String> args) throws IOException {
+ CommandFormat cf =
+ new CommandFormat(1, Integer.MAX_VALUE, OPTION_FOLLOW_LINK,
+ OPTION_FOLLOW_ARG_LINK, null);
+ cf.parse(args);
+
+ if (cf.getOpt(OPTION_FOLLOW_LINK)) {
+ getOptions().setFollowLink(true);
+ } else if (cf.getOpt(OPTION_FOLLOW_ARG_LINK)) {
+ getOptions().setFollowArgLink(true);
+ }
+
+ // search for first non-path argument (ie starts with a "-") and capture and
+ // remove the remaining arguments as expressions
+ LinkedList<String> expressionArgs = new LinkedList<String>();
+ Iterator<String> it = args.iterator();
+ boolean isPath = true;
+ while (it.hasNext()) {
+ String arg = it.next();
+ if (isPath) {
+ if (arg.startsWith("-")) {
+ isPath = false;
+ }
+ }
+ if (!isPath) {
+ expressionArgs.add(arg);
+ it.remove();
+ }
+ }
+
+ if (args.isEmpty()) {
+ args.add(Path.CUR_DIR);
+ }
+
+ Expression expression = parseExpression(expressionArgs);
+ if (!expression.isAction()) {
+ Expression and = getExpression(And.class);
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(getExpression(Print.class));
+ children.add(expression);
+ and.addChildren(children);
+ expression = and;
+ }
+
+ setRootExpression(expression);
+ }
+
+ /**
+ * Set the root expression for this find.
+ *
+ * @param expression
+ */
+ @InterfaceAudience.Private
+ void setRootExpression(Expression expression) {
+ this.rootExpression = expression;
+ }
+
+ /**
+ * Return the root expression for this find.
+ *
+ * @return the root expression
+ */
+ @InterfaceAudience.Private
+ Expression getRootExpression() {
+ return this.rootExpression;
+ }
+
+ /** Returns the current find options, creating them if necessary. */
+ @InterfaceAudience.Private
+ FindOptions getOptions() {
+ if (options == null) {
+ options = createOptions();
+ }
+ return options;
+ }
+
+ /** Create a new set of find options. */
+ private FindOptions createOptions() {
+ FindOptions options = new FindOptions();
+ options.setOut(out);
+ options.setErr(err);
+ options.setIn(System.in);
+ options.setCommandFactory(getCommandFactory());
+ options.setConfiguration(getConf());
+ return options;
+ }
+
+ /** Add the {@link PathData} item to the stop set. */
+ private void addStop(PathData item) {
+ stopPaths.add(item.path);
+ }
+
+ /** Returns true if the {@link PathData} item is in the stop set. */
+ private boolean isStop(PathData item) {
+ return stopPaths.contains(item.path);
+ }
+
+ /**
+ * Parse a list of arguments to to extract the {@link Expression} elements.
+ * The input Deque will be modified to remove the used elements.
+ *
+ * @param args arguments to be parsed
+ * @return list of {@link Expression} elements applicable to this command
+ * @throws IOException if list can not be parsed
+ */
+ private Expression parseExpression(Deque<String> args) throws IOException {
+ Deque<Expression> primaries = new LinkedList<Expression>();
+ Deque<Expression> operators = new LinkedList<Expression>();
+ Expression prevExpr = getExpression(And.class);
+ while (!args.isEmpty()) {
+ String arg = args.pop();
+ if ("(".equals(arg)) {
+ Expression expr = parseExpression(args);
+ primaries.add(expr);
+ prevExpr = new BaseExpression() {
+ @Override
+ public Result apply(PathData item, int depth) throws IOException {
+ return Result.PASS;
+ }
+ }; // stub the previous expression to be a non-op
+ } else if (")".equals(arg)) {
+ break;
+ } else if (isExpression(arg)) {
+ Expression expr = getExpression(arg);
+ expr.addArguments(args);
+ if (expr.isOperator()) {
+ while (!operators.isEmpty()) {
+ if (operators.peek().getPrecedence() >= expr.getPrecedence()) {
+ Expression op = operators.pop();
+ op.addChildren(primaries);
+ primaries.push(op);
+ } else {
+ break;
+ }
+ }
+ operators.push(expr);
+ } else {
+ if (!prevExpr.isOperator()) {
+ Expression and = getExpression(And.class);
+ while (!operators.isEmpty()) {
+ if (operators.peek().getPrecedence() >= and.getPrecedence()) {
+ Expression op = operators.pop();
+ op.addChildren(primaries);
+ primaries.push(op);
+ } else {
+ break;
+ }
+ }
+ operators.push(and);
+ }
+ primaries.push(expr);
+ }
+ prevExpr = expr;
+ } else {
+ throw new IOException("Unexpected argument: " + arg);
+ }
+ }
+
+ while (!operators.isEmpty()) {
+ Expression operator = operators.pop();
+ operator.addChildren(primaries);
+ primaries.push(operator);
+ }
+
+ return primaries.isEmpty() ? getExpression(Print.class) : primaries.pop();
+ }
+
+ /** Returns true if the target is an ancestor of the source. */
+ private boolean isAncestor(PathData source, PathData target) {
+ for (Path parent = source.path; (parent != null) && !parent.isRoot();
+ parent = parent.getParent()) {
+ if (parent.equals(target.path)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ protected void recursePath(PathData item) throws IOException {
+ if (isStop(item)) {
+ // this item returned a stop result so don't recurse any further
+ return;
+ }
+ if (getDepth() >= getOptions().getMaxDepth()) {
+ // reached the maximum depth so don't got any further.
+ return;
+ }
+ if (item.stat.isSymlink() && getOptions().isFollowLink()) {
+ PathData linkedItem =
+ new PathData(item.stat.getSymlink().toString(), getConf());
+ if (isAncestor(item, linkedItem)) {
+ getOptions().getErr().println(
+ "Infinite loop ignored: " + item.toString() + " -> "
+ + linkedItem.toString());
+ return;
+ }
+ if (linkedItem.exists) {
+ item = linkedItem;
+ }
+ }
+ if (item.stat.isDirectory()) {
+ super.recursePath(item);
+ }
+ }
+
+ @Override
+ protected boolean isPathRecursable(PathData item) throws IOException {
+ if (item.stat.isDirectory()) {
+ return true;
+ }
+ if (item.stat.isSymlink()) {
+ PathData linkedItem =
+ new PathData(item.fs.resolvePath(item.stat.getSymlink()).toString(),
+ getConf());
+ if (linkedItem.stat.isDirectory()) {
+ if (getOptions().isFollowLink()) {
+ return true;
+ }
+ if (getOptions().isFollowArgLink() && (getDepth() == 0)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ if (getOptions().isDepthFirst()) {
+ // depth first so leave until post processing
+ return;
+ }
+ applyItem(item);
+ }
+
+ @Override
+ protected void postProcessPath(PathData item) throws IOException {
+ if (!getOptions().isDepthFirst()) {
+ // not depth first so already processed
+ return;
+ }
+ applyItem(item);
+ }
+
+ private void applyItem(PathData item) throws IOException {
+ if (getDepth() >= getOptions().getMinDepth()) {
+ Result result = getRootExpression().apply(item, getDepth());
+ if (Result.STOP.equals(result)) {
+ addStop(item);
+ }
+ }
+ }
+
+ @Override
+ protected void processArguments(LinkedList<PathData> args)
+ throws IOException {
+ Expression expr = getRootExpression();
+ expr.setOptions(getOptions());
+ expr.prepare();
+ super.processArguments(args);
+ expr.finish();
+ }
+
+ /** Gets a named expression from the factory. */
+ private Expression getExpression(String expressionName) {
+ return ExpressionFactory.getExpressionFactory().getExpression(
+ expressionName, getConf());
+ }
+
+ /** Gets an instance of an expression from the factory. */
+ private Expression getExpression(
+ Class<? extends Expression> expressionClass) {
+ return ExpressionFactory.getExpressionFactory().createExpression(
+ expressionClass, getConf());
+ }
+
+ /** Asks the factory whether an expression is recognized. */
+ private boolean isExpression(String expressionName) {
+ return ExpressionFactory.getExpressionFactory()
+ .isExpression(expressionName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java
new file mode 100644
index 0000000..b0f1be5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/FindOptions.java
@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.util.Date;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.shell.CommandFactory;
+
+/**
+ * Options to be used by the {@link Find} command and its {@link Expression}s.
+ */
+public class FindOptions {
+ /** Output stream to be used. */
+ private PrintStream out;
+
+ /** Error stream to be used. */
+ private PrintStream err;
+
+ /** Input stream to be used. */
+ private InputStream in;
+
+ /**
+ * Indicates whether the expression should be applied to the directory tree
+ * depth first.
+ */
+ private boolean depthFirst = false;
+
+ /** Indicates whether symbolic links should be followed. */
+ private boolean followLink = false;
+
+ /**
+ * Indicates whether symbolic links specified as command arguments should be
+ * followed.
+ */
+ private boolean followArgLink = false;
+
+ /** Start time of the find process. */
+ private long startTime = new Date().getTime();
+
+ /**
+ * Depth at which to start applying expressions.
+ */
+ private int minDepth = 0;
+
+ /**
+ * Depth at which to stop applying expressions.
+ */
+ private int maxDepth = Integer.MAX_VALUE;
+
+ /** Factory for retrieving command classes. */
+ private CommandFactory commandFactory;
+
+ /** Configuration object. */
+ private Configuration configuration = new Configuration();
+
+ /**
+ * Sets the output stream to be used.
+ *
+ * @param out output stream to be used
+ */
+ public void setOut(PrintStream out) {
+ this.out = out;
+ }
+
+ /**
+ * Returns the output stream to be used.
+ *
+ * @return output stream to be used
+ */
+ public PrintStream getOut() {
+ return this.out;
+ }
+
+ /**
+ * Sets the error stream to be used.
+ *
+ * @param err error stream to be used
+ */
+ public void setErr(PrintStream err) {
+ this.err = err;
+ }
+
+ /**
+ * Returns the error stream to be used.
+ *
+ * @return error stream to be used
+ */
+ public PrintStream getErr() {
+ return this.err;
+ }
+
+ /**
+ * Sets the input stream to be used.
+ *
+ * @param in input stream to be used
+ */
+ public void setIn(InputStream in) {
+ this.in = in;
+ }
+
+ /**
+ * Returns the input stream to be used.
+ *
+ * @return input stream to be used
+ */
+ public InputStream getIn() {
+ return this.in;
+ }
+
+ /**
+ * Sets flag indicating whether the expression should be applied to the
+ * directory tree depth first.
+ *
+ * @param depthFirst true indicates depth first traversal
+ */
+ public void setDepthFirst(boolean depthFirst) {
+ this.depthFirst = depthFirst;
+ }
+
+ /**
+ * Should directory tree be traversed depth first?
+ *
+ * @return true indicate depth first traversal
+ */
+ public boolean isDepthFirst() {
+ return this.depthFirst;
+ }
+
+ /**
+ * Sets flag indicating whether symbolic links should be followed.
+ *
+ * @param followLink true indicates follow links
+ */
+ public void setFollowLink(boolean followLink) {
+ this.followLink = followLink;
+ }
+
+ /**
+ * Should symbolic links be follows?
+ *
+ * @return true indicates links should be followed
+ */
+ public boolean isFollowLink() {
+ return this.followLink;
+ }
+
+ /**
+ * Sets flag indicating whether command line symbolic links should be
+ * followed.
+ *
+ * @param followArgLink true indicates follow links
+ */
+ public void setFollowArgLink(boolean followArgLink) {
+ this.followArgLink = followArgLink;
+ }
+
+ /**
+ * Should command line symbolic links be follows?
+ *
+ * @return true indicates links should be followed
+ */
+ public boolean isFollowArgLink() {
+ return this.followArgLink;
+ }
+
+ /**
+ * Returns the start time of this {@link Find} command.
+ *
+ * @return start time (in milliseconds since epoch)
+ */
+ public long getStartTime() {
+ return this.startTime;
+ }
+
+ /**
+ * Set the start time of this {@link Find} command.
+ *
+ * @param time start time (in milliseconds since epoch)
+ */
+ public void setStartTime(long time) {
+ this.startTime = time;
+ }
+
+ /**
+ * Returns the minimum depth for applying expressions.
+ *
+ * @return min depth
+ */
+ public int getMinDepth() {
+ return this.minDepth;
+ }
+
+ /**
+ * Sets the minimum depth for applying expressions.
+ *
+ * @param minDepth minimum depth
+ */
+ public void setMinDepth(int minDepth) {
+ this.minDepth = minDepth;
+ }
+
+ /**
+ * Returns the maximum depth for applying expressions.
+ *
+ * @return maximum depth
+ */
+ public int getMaxDepth() {
+ return this.maxDepth;
+ }
+
+ /**
+ * Sets the maximum depth for applying expressions.
+ *
+ * @param maxDepth maximum depth
+ */
+ public void setMaxDepth(int maxDepth) {
+ this.maxDepth = maxDepth;
+ }
+
+ /**
+ * Set the command factory.
+ *
+ * @param factory {@link CommandFactory}
+ */
+ public void setCommandFactory(CommandFactory factory) {
+ this.commandFactory = factory;
+ }
+
+ /**
+ * Return the command factory.
+ *
+ * @return {@link CommandFactory}
+ */
+ public CommandFactory getCommandFactory() {
+ return this.commandFactory;
+ }
+
+ /**
+ * Set the {@link Configuration}
+ *
+ * @param configuration {@link Configuration}
+ */
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * Return the {@link Configuration} return configuration {@link Configuration}
+ */
+ public Configuration getConfiguration() {
+ return this.configuration;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Name.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Name.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Name.java
new file mode 100644
index 0000000..88314c6
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Name.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.io.IOException;
+import java.util.Deque;
+
+import org.apache.hadoop.fs.GlobPattern;
+import org.apache.hadoop.fs.shell.PathData;
+
+/**
+ * Implements the -name expression for the
+ * {@link org.apache.hadoop.fs.shell.find.Find} command.
+ */
+final class Name extends BaseExpression {
+ /** Registers this expression with the specified factory. */
+ public static void registerExpression(ExpressionFactory factory)
+ throws IOException {
+ factory.addClass(Name.class, "-name");
+ factory.addClass(Iname.class, "-iname");
+ }
+
+ private static final String[] USAGE = { "-name pattern", "-iname pattern" };
+ private static final String[] HELP = {
+ "Evaluates as true if the basename of the file matches the",
+ "pattern using standard file system globbing.",
+ "If -iname is used then the match is case insensitive." };
+ private GlobPattern globPattern;
+ private boolean caseSensitive = true;
+
+ /** Creates a case sensitive name expression. */
+ public Name() {
+ this(true);
+ }
+
+ /**
+ * Construct a Name {@link Expression} with a specified case sensitivity.
+ *
+ * @param caseSensitive if true the comparisons are case sensitive.
+ */
+ private Name(boolean caseSensitive) {
+ super();
+ setUsage(USAGE);
+ setHelp(HELP);
+ setCaseSensitive(caseSensitive);
+ }
+
+ private void setCaseSensitive(boolean caseSensitive) {
+ this.caseSensitive = caseSensitive;
+ }
+
+ @Override
+ public void addArguments(Deque<String> args) {
+ addArguments(args, 1);
+ }
+
+ @Override
+ public void prepare() throws IOException {
+ String argPattern = getArgument(1);
+ if (!caseSensitive) {
+ argPattern = argPattern.toLowerCase();
+ }
+ globPattern = new GlobPattern(argPattern);
+ }
+
+ @Override
+ public Result apply(PathData item, int depth) throws IOException {
+ String name = getPath(item).getName();
+ if (!caseSensitive) {
+ name = name.toLowerCase();
+ }
+ if (globPattern.matches(name)) {
+ return Result.PASS;
+ } else {
+ return Result.FAIL;
+ }
+ }
+
+ /** Case insensitive version of the -name expression. */
+ static class Iname extends FilterExpression {
+ public Iname() {
+ super(new Name(false));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Print.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Print.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Print.java
new file mode 100644
index 0000000..ae99779
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Print.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.shell.PathData;
+
+/**
+ * Implements the -print expression for the
+ * {@link org.apache.hadoop.fs.shell.find.Find} command.
+ */
+final class Print extends BaseExpression {
+ /** Registers this expression with the specified factory. */
+ public static void registerExpression(ExpressionFactory factory)
+ throws IOException {
+ factory.addClass(Print.class, "-print");
+ factory.addClass(Print0.class, "-print0");
+ }
+
+ private static final String[] USAGE = { "-print", "-print0" };
+ private static final String[] HELP = {
+ "Always evaluates to true. Causes the current pathname to be",
+ "written to standard output followed by a newline. If the -print0",
+ "expression is used then an ASCII NULL character is appended rather",
+ "than a newline." };
+
+ private final String suffix;
+
+ public Print() {
+ this("\n");
+ }
+
+ /**
+ * Construct a Print {@link Expression} with the specified suffix.
+ */
+ private Print(String suffix) {
+ super();
+ setUsage(USAGE);
+ setHelp(HELP);
+ this.suffix = suffix;
+ }
+
+ @Override
+ public Result apply(PathData item, int depth) throws IOException {
+ getOptions().getOut().print(item.toString() + suffix);
+ return Result.PASS;
+ }
+
+ @Override
+ public boolean isAction() {
+ return true;
+ }
+
+ /** Implements the -print0 expression. */
+ final static class Print0 extends FilterExpression {
+ public Print0() {
+ super(new Print("\0"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java
new file mode 100644
index 0000000..2ef9cb4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/find/Result.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+public final class Result {
+ /** Result indicating {@link Expression} processing should continue. */
+ public static final Result PASS = new Result(true, true);
+ /** Result indicating {@link Expression} processing should stop. */
+ public static final Result FAIL = new Result(false, true);
+ /**
+ * Result indicating {@link Expression} processing should not descend any more
+ * directories.
+ */
+ public static final Result STOP = new Result(true, false);
+ private boolean descend;
+ private boolean success;
+
+ private Result(boolean success, boolean recurse) {
+ this.success = success;
+ this.descend = recurse;
+ }
+
+ /** Should further directories be descended. */
+ public boolean isDescend() {
+ return this.descend;
+ }
+
+ /** Should processing continue. */
+ public boolean isPass() {
+ return this.success;
+ }
+
+ /** Returns the combination of this and another result. */
+ public Result combine(Result other) {
+ return new Result(this.isPass() && other.isPass(), this.isDescend()
+ && other.isDescend());
+ }
+
+ /** Negate this result. */
+ public Result negate() {
+ return new Result(!this.isPass(), this.isDescend());
+ }
+
+ @Override
+ public String toString() {
+ return "success=" + isPass() + "; recurse=" + isDescend();
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (descend ? 1231 : 1237);
+ result = prime * result + (success ? 1231 : 1237);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ Result other = (Result) obj;
+ if (descend != other.descend)
+ return false;
+ if (success != other.success)
+ return false;
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
index abc4643..1a9618c 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
@@ -232,6 +232,49 @@ expunge
Empty the Trash. Refer to the {{{../hadoop-hdfs/HdfsDesign.html}
HDFS Architecture Guide}} for more information on the Trash feature.
+find
+
+ Usage: <<<hdfs dfs -find <path> ... <expression> ... >>>
+
+ Finds all files that match the specified expression and applies selected
+ actions to them. If no <path> is specified then defaults to the current
+ working directory. If no expression is specified then defaults to -print.
+
+ The following primary expressions are recognised:
+
+ * -name pattern \
+ -iname pattern
+
+ Evaluates as true if the basename of the file matches the pattern using
+ standard file system globbing. If -iname is used then the match is case
+ insensitive.
+
+ * -print \
+ -print0
+
+ Always evaluates to true. Causes the current pathname to be written to
+ standard output. If the -print0 expression is used then an ASCII NULL
+ character is appended.
+
+ The following operators are recognised:
+
+ * expression -a expression \
+ expression -and expression \
+ expression expression
+
+ Logical AND operator for joining two expressions. Returns true if both
+ child expressions return true. Implied by the juxtaposition of two
+ expressions and so does not need to be explicitly specified. The second
+ expression will not be applied if the first fails.
+
+ Example:
+
+ <<<hdfs dfs -find / -name test -print>>>
+
+ Exit Code:
+
+ Returns 0 on success and -1 on error.
+
get
Usage: <<<hdfs dfs -get [-ignorecrc] [-crc] <src> <localdst> >>>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/MockFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/MockFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/MockFileSystem.java
new file mode 100644
index 0000000..44abd23
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/MockFileSystem.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * A mock {@link FileSystem} for use with the {@link Find} unit tests. Usage:
+ * FileSystem mockFs = MockFileSystem.setup(); Methods in the mockFs can then be
+ * mocked out by the test script. The {@link Configuration} can be accessed by
+ * mockFs.getConf(); The following methods are fixed within the class: -
+ * {@link FileSystem#initialize(URI,Configuration)} blank stub -
+ * {@link FileSystem#makeQualified(Path)} returns the passed in {@link Path} -
+ * {@link FileSystem#getWorkingDirectory} returns new Path("/") -
+ * {@link FileSystem#resolvePath(Path)} returns the passed in {@link Path}
+ */
+class MockFileSystem extends FilterFileSystem {
+ private static FileSystem mockFs = null;
+
+ /** Setup and return the underlying {@link FileSystem} mock */
+ static FileSystem setup() throws IOException {
+ if (mockFs == null) {
+ mockFs = mock(FileSystem.class);
+ }
+ reset(mockFs);
+ Configuration conf = new Configuration();
+ conf.set("fs.defaultFS", "mockfs:///");
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+ when(mockFs.getConf()).thenReturn(conf);
+ return mockFs;
+ }
+
+ private MockFileSystem() {
+ super(mockFs);
+ }
+
+ @Override
+ public void initialize(URI uri, Configuration conf) {
+ }
+
+ @Override
+ public Path makeQualified(Path path) {
+ return path;
+ }
+
+ @Override
+ public FileStatus[] globStatus(Path pathPattern) throws IOException {
+ return fs.globStatus(pathPattern);
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return new Path("/");
+ }
+
+ @Override
+ public Path resolvePath(final Path p) throws IOException {
+ return p;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestAnd.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestAnd.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestAnd.java
new file mode 100644
index 0000000..d82a25e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestAnd.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell.find;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.io.IOException;
+import java.util.Deque;
+import java.util.LinkedList;
+
+import org.apache.hadoop.fs.shell.PathData;
+import org.junit.Test;
+
+public class TestAnd {
+
+ // test all expressions passing
+ @Test(timeout = 1000)
+ public void testPass() throws IOException {
+ And and = new And();
+
+ PathData pathData = mock(PathData.class);
+
+ Expression first = mock(Expression.class);
+ when(first.apply(pathData, -1)).thenReturn(Result.PASS);
+
+ Expression second = mock(Expression.class);
+ when(second.apply(pathData, -1)).thenReturn(Result.PASS);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ assertEquals(Result.PASS, and.apply(pathData, -1));
+ verify(first).apply(pathData, -1);
+ verify(second).apply(pathData, -1);
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+
+ // test the first expression failing
+ @Test(timeout = 1000)
+ public void testFailFirst() throws IOException {
+ And and = new And();
+
+ PathData pathData = mock(PathData.class);
+
+ Expression first = mock(Expression.class);
+ when(first.apply(pathData, -1)).thenReturn(Result.FAIL);
+
+ Expression second = mock(Expression.class);
+ when(second.apply(pathData, -1)).thenReturn(Result.PASS);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ assertEquals(Result.FAIL, and.apply(pathData, -1));
+ verify(first).apply(pathData, -1);
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+
+ // test the second expression failing
+ @Test(timeout = 1000)
+ public void testFailSecond() throws IOException {
+ And and = new And();
+
+ PathData pathData = mock(PathData.class);
+
+ Expression first = mock(Expression.class);
+ when(first.apply(pathData, -1)).thenReturn(Result.PASS);
+
+ Expression second = mock(Expression.class);
+ when(second.apply(pathData, -1)).thenReturn(Result.FAIL);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ assertEquals(Result.FAIL, and.apply(pathData, -1));
+ verify(first).apply(pathData, -1);
+ verify(second).apply(pathData, -1);
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+
+ // test both expressions failing
+ @Test(timeout = 1000)
+ public void testFailBoth() throws IOException {
+ And and = new And();
+
+ PathData pathData = mock(PathData.class);
+
+ Expression first = mock(Expression.class);
+ when(first.apply(pathData, -1)).thenReturn(Result.FAIL);
+
+ Expression second = mock(Expression.class);
+ when(second.apply(pathData, -1)).thenReturn(Result.FAIL);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ assertEquals(Result.FAIL, and.apply(pathData, -1));
+ verify(first).apply(pathData, -1);
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+
+ // test the first expression stopping
+ @Test(timeout = 1000)
+ public void testStopFirst() throws IOException {
+ And and = new And();
+
+ PathData pathData = mock(PathData.class);
+
+ Expression first = mock(Expression.class);
+ when(first.apply(pathData, -1)).thenReturn(Result.STOP);
+
+ Expression second = mock(Expression.class);
+ when(second.apply(pathData, -1)).thenReturn(Result.PASS);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ assertEquals(Result.STOP, and.apply(pathData, -1));
+ verify(first).apply(pathData, -1);
+ verify(second).apply(pathData, -1);
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+
+ // test the second expression stopping
+ @Test(timeout = 1000)
+ public void testStopSecond() throws IOException {
+ And and = new And();
+
+ PathData pathData = mock(PathData.class);
+
+ Expression first = mock(Expression.class);
+ when(first.apply(pathData, -1)).thenReturn(Result.PASS);
+
+ Expression second = mock(Expression.class);
+ when(second.apply(pathData, -1)).thenReturn(Result.STOP);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ assertEquals(Result.STOP, and.apply(pathData, -1));
+ verify(first).apply(pathData, -1);
+ verify(second).apply(pathData, -1);
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+
+ // test first expression stopping and second failing
+ @Test(timeout = 1000)
+ public void testStopFail() throws IOException {
+ And and = new And();
+
+ PathData pathData = mock(PathData.class);
+
+ Expression first = mock(Expression.class);
+ when(first.apply(pathData, -1)).thenReturn(Result.STOP);
+
+ Expression second = mock(Expression.class);
+ when(second.apply(pathData, -1)).thenReturn(Result.FAIL);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ assertEquals(Result.STOP.combine(Result.FAIL), and.apply(pathData, -1));
+ verify(first).apply(pathData, -1);
+ verify(second).apply(pathData, -1);
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+
+ // test setOptions is called on child
+ @Test(timeout = 1000)
+ public void testSetOptions() throws IOException {
+ And and = new And();
+ Expression first = mock(Expression.class);
+ Expression second = mock(Expression.class);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ FindOptions options = mock(FindOptions.class);
+ and.setOptions(options);
+ verify(first).setOptions(options);
+ verify(second).setOptions(options);
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+
+ // test prepare is called on child
+ @Test(timeout = 1000)
+ public void testPrepare() throws IOException {
+ And and = new And();
+ Expression first = mock(Expression.class);
+ Expression second = mock(Expression.class);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ and.prepare();
+ verify(first).prepare();
+ verify(second).prepare();
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+
+ // test finish is called on child
+ @Test(timeout = 1000)
+ public void testFinish() throws IOException {
+ And and = new And();
+ Expression first = mock(Expression.class);
+ Expression second = mock(Expression.class);
+
+ Deque<Expression> children = new LinkedList<Expression>();
+ children.add(second);
+ children.add(first);
+ and.addChildren(children);
+
+ and.finish();
+ verify(first).finish();
+ verify(second).finish();
+ verifyNoMoreInteractions(first);
+ verifyNoMoreInteractions(second);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFilterExpression.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFilterExpression.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFilterExpression.java
new file mode 100644
index 0000000..5986a06
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFilterExpression.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.io.IOException;
+import java.util.Deque;
+
+import org.apache.hadoop.fs.shell.PathData;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestFilterExpression {
+ private Expression expr;
+ private FilterExpression test;
+
+ @Before
+ public void setup() {
+ expr = mock(Expression.class);
+ test = new FilterExpression(expr) {
+ };
+ }
+
+ // test that the child expression is correctly set
+ @Test(timeout = 1000)
+ public void expression() throws IOException {
+ assertEquals(expr, test.expression);
+ }
+
+ // test that setOptions method is called
+ @Test(timeout = 1000)
+ public void setOptions() throws IOException {
+ FindOptions options = mock(FindOptions.class);
+ test.setOptions(options);
+ verify(expr).setOptions(options);
+ verifyNoMoreInteractions(expr);
+ }
+
+ // test the apply method is called and the result returned
+ @Test(timeout = 1000)
+ public void apply() throws IOException {
+ PathData item = mock(PathData.class);
+ when(expr.apply(item, -1)).thenReturn(Result.PASS).thenReturn(Result.FAIL);
+ assertEquals(Result.PASS, test.apply(item, -1));
+ assertEquals(Result.FAIL, test.apply(item, -1));
+ verify(expr, times(2)).apply(item, -1);
+ verifyNoMoreInteractions(expr);
+ }
+
+ // test that the finish method is called
+ @Test(timeout = 1000)
+ public void finish() throws IOException {
+ test.finish();
+ verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+ }
+
+ // test that the getUsage method is called
+ @Test(timeout = 1000)
+ public void getUsage() {
+ String[] usage = new String[] { "Usage 1", "Usage 2", "Usage 3" };
+ when(expr.getUsage()).thenReturn(usage);
+ assertArrayEquals(usage, test.getUsage());
+ verify(expr).getUsage();
+ verifyNoMoreInteractions(expr);
+ }
+
+ // test that the getHelp method is called
+ @Test(timeout = 1000)
+ public void getHelp() {
+ String[] help = new String[] { "Help 1", "Help 2", "Help 3" };
+ when(expr.getHelp()).thenReturn(help);
+ assertArrayEquals(help, test.getHelp());
+ verify(expr).getHelp();
+ verifyNoMoreInteractions(expr);
+ }
+
+ // test that the isAction method is called
+ @Test(timeout = 1000)
+ public void isAction() {
+ when(expr.isAction()).thenReturn(true).thenReturn(false);
+ assertTrue(test.isAction());
+ assertFalse(test.isAction());
+ verify(expr, times(2)).isAction();
+ verifyNoMoreInteractions(expr);
+ }
+
+ // test that the isOperator method is called
+ @Test(timeout = 1000)
+ public void isOperator() {
+ when(expr.isAction()).thenReturn(true).thenReturn(false);
+ assertTrue(test.isAction());
+ assertFalse(test.isAction());
+ verify(expr, times(2)).isAction();
+ verifyNoMoreInteractions(expr);
+ }
+
+ // test that the getPrecedence method is called
+ @Test(timeout = 1000)
+ public void getPrecedence() {
+ int precedence = 12345;
+ when(expr.getPrecedence()).thenReturn(precedence);
+ assertEquals(precedence, test.getPrecedence());
+ verify(expr).getPrecedence();
+ verifyNoMoreInteractions(expr);
+ }
+
+ // test that the addChildren method is called
+ @Test(timeout = 1000)
+ public void addChildren() {
+ @SuppressWarnings("unchecked")
+ Deque<Expression> expressions = mock(Deque.class);
+ test.addChildren(expressions);
+ verify(expr).addChildren(expressions);
+ verifyNoMoreInteractions(expr);
+ }
+
+ // test that the addArguments method is called
+ @Test(timeout = 1000)
+ public void addArguments() {
+ @SuppressWarnings("unchecked")
+ Deque<String> args = mock(Deque.class);
+ test.addArguments(args);
+ verify(expr).addArguments(args);
+ verifyNoMoreInteractions(expr);
+ }
+}
[25/45] hadoop git commit: HADOOP-11150. hadoop command should show
the reason on failure by invalid COMMAND or CLASSNAME (Masatake Iwasaki via
aw)
Posted by vi...@apache.org.
HADOOP-11150. hadoop command should show the reason on failure by invalid COMMAND or CLASSNAME (Masatake Iwasaki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aee68b67
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aee68b67
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aee68b67
Branch: refs/heads/HDFS-EC
Commit: aee68b67f3894e19757f6e4f3ffb6ca10b9a0d64
Parents: b650e61
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Nov 13 12:08:03 2014 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Nov 13 12:08:03 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/main/bin/hadoop-functions.sh | 3 +++
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aee68b67/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f3a0237..1a8c41d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -138,6 +138,9 @@ Trunk (Unreleased)
HADOOP-11025. hadoop-daemons.sh should just call hdfs directly (Masatake
Iwasaki via aw)
+ HADOOP-11150. hadoop command should show the reason on failure by
+ invalid COMMAND or CLASSNAME (Masatake Iwasaki via aw)
+
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aee68b67/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 4f75a7c..b5d4b1c 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -286,6 +286,9 @@ function hadoop_validate_classname
shift 1
if [[ ! ${class} =~ \. ]]; then
+ # assuming the arg is typo of command if it does not conatain ".".
+ # class belonging to no package is not allowed as a result.
+ hadoop_error "ERROR: ${class} is not COMMAND nor fully qualified CLASSNAME."
return 1
fi
return 0
[02/45] hadoop git commit: YARN-2780. Log aggregated resource
allocation in rm-appsummary.log. Contributed by Eric Payne
Posted by vi...@apache.org.
YARN-2780. Log aggregated resource allocation in rm-appsummary.log. Contributed by Eric Payne
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8aefa5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8aefa5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8aefa5e
Branch: refs/heads/HDFS-EC
Commit: f8aefa5e9c8c6d2817205b5ed8d914db31f56ae7
Parents: 53f64ee
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Nov 12 17:01:15 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Nov 12 17:01:15 2014 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/RMAppManager.java | 9 ++++++++-
.../yarn/server/resourcemanager/TestAppManager.java | 12 +++++++++++-
3 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8aefa5e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c3bdf3d..05bb8f6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -68,6 +68,9 @@ Release 2.7.0 - UNRELEASED
YARN-570. Time strings are formated in different timezone.
(Akira Ajisaka and Peng Zhang via kasha)
+ YARN-2780. Log aggregated resource allocation in rm-appsummary.log (Eric
+ Payne via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8aefa5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 02c6d2f..ab8df62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRecoverEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -155,6 +156,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
trackingUrl = attempt.getTrackingUrl();
host = attempt.getHost();
}
+ RMAppMetrics metrics = app.getRMAppMetrics();
SummaryBuilder summary = new SummaryBuilder()
.add("appId", app.getApplicationId())
.add("name", app.getName())
@@ -165,7 +167,12 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
.add("appMasterHost", host)
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
- .add("finalStatus", app.getFinalApplicationStatus());
+ .add("finalStatus", app.getFinalApplicationStatus())
+ .add("memorySeconds", metrics.getMemorySeconds())
+ .add("vcoreSeconds", metrics.getVcoreSeconds())
+ .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
+ .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
+ .add("preemptedResources", metrics.getResourcePreempted());
return summary;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8aefa5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index e146611..6b3eea2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
@@ -572,6 +573,10 @@ public class TestAppManager{
when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
when(app.getState()).thenReturn(RMAppState.RUNNING);
+ RMAppMetrics metrics =
+ new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64);
+ when(app.getRMAppMetrics()).thenReturn(metrics);
+
RMAppManager.ApplicationSummary.SummaryBuilder summary =
new RMAppManager.ApplicationSummary().createAppSummary(app);
String msg = summary.toString();
@@ -583,7 +588,12 @@ public class TestAppManager{
Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
- }
+ Assert.assertTrue(msg.contains("memorySeconds=16384"));
+ Assert.assertTrue(msg.contains("vcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
+ Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
+ Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
+ }
private static ResourceScheduler mockResourceScheduler() {
ResourceScheduler scheduler = mock(ResourceScheduler.class);
[22/45] hadoop git commit: HADOOP-8989. hadoop fs -find feature
(Jonathan Allen via aw)
Posted by vi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFind.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFind.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFind.java
new file mode 100644
index 0000000..7d79420
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestFind.java
@@ -0,0 +1,900 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+import static org.mockito.Matchers.*;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.PathData;
+import org.apache.hadoop.fs.shell.find.BaseExpression;
+import org.apache.hadoop.fs.shell.find.Expression;
+import org.apache.hadoop.fs.shell.find.Find;
+import org.apache.hadoop.fs.shell.find.FindOptions;
+import org.apache.hadoop.fs.shell.find.Result;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InOrder;
+
+public class TestFind {
+ private static FileSystem mockFs;
+ private static Configuration conf;
+
+ @Before
+ public void setup() throws IOException {
+ mockFs = MockFileSystem.setup();
+ conf = mockFs.getConf();
+ }
+
+ // check follow link option is recognized
+ @Test(timeout = 1000)
+ public void processOptionsFollowLink() throws IOException {
+ Find find = new Find();
+ String args = "-L path";
+ find.processOptions(getArgs(args));
+ assertTrue(find.getOptions().isFollowLink());
+ assertFalse(find.getOptions().isFollowArgLink());
+ }
+
+ // check follow arg link option is recognized
+ @Test(timeout = 1000)
+ public void processOptionsFollowArgLink() throws IOException {
+ Find find = new Find();
+ String args = "-H path";
+ find.processOptions(getArgs(args));
+ assertFalse(find.getOptions().isFollowLink());
+ assertTrue(find.getOptions().isFollowArgLink());
+ }
+
+ // check follow arg link option is recognized
+ @Test(timeout = 1000)
+ public void processOptionsFollowLinkFollowArgLink() throws IOException {
+ Find find = new Find();
+ String args = "-L -H path";
+ find.processOptions(getArgs(args));
+ assertTrue(find.getOptions().isFollowLink());
+
+ // follow link option takes precedence over follow arg link
+ assertFalse(find.getOptions().isFollowArgLink());
+ }
+
+ // check options and expressions are stripped from args leaving paths
+ @Test(timeout = 1000)
+ public void processOptionsExpression() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+
+ String paths = "path1 path2 path3";
+ String args = "-L -H " + paths + " -print -name test";
+ LinkedList<String> argsList = getArgs(args);
+ find.processOptions(argsList);
+ LinkedList<String> pathList = getArgs(paths);
+ assertEquals(pathList, argsList);
+ }
+
+ // check print is used as the default expression
+ @Test(timeout = 1000)
+ public void processOptionsNoExpression() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+ String args = "path";
+ String expected = "Print(;)";
+ find.processOptions(getArgs(args));
+ Expression expression = find.getRootExpression();
+ assertEquals(expected, expression.toString());
+ }
+
+ // check unknown options are rejected
+ @Test(timeout = 1000)
+ public void processOptionsUnknown() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+ String args = "path -unknown";
+ try {
+ find.processOptions(getArgs(args));
+ fail("Unknown expression not caught");
+ } catch (IOException e) {
+ }
+ }
+
+ // check unknown options are rejected when mixed with known options
+ @Test(timeout = 1000)
+ public void processOptionsKnownUnknown() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+ String args = "path -print -unknown -print";
+ try {
+ find.processOptions(getArgs(args));
+ fail("Unknown expression not caught");
+ } catch (IOException e) {
+ }
+ }
+
+ // check no path defaults to current working directory
+ @Test(timeout = 1000)
+ public void processOptionsNoPath() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+ String args = "-print";
+
+ LinkedList<String> argsList = getArgs(args);
+ find.processOptions(argsList);
+ assertEquals(Collections.singletonList(Path.CUR_DIR), argsList);
+ }
+
+ // check -name is handled correctly
+ @Test(timeout = 1000)
+ public void processOptionsName() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+ String args = "path -name namemask";
+ String expected = "And(;Name(namemask;),Print(;))";
+ find.processOptions(getArgs(args));
+ Expression expression = find.getRootExpression();
+ assertEquals(expected, expression.toString());
+ }
+
+ // check -iname is handled correctly
+ @Test(timeout = 1000)
+ public void processOptionsIname() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+ String args = "path -iname namemask";
+ String expected = "And(;Iname-Name(namemask;),Print(;))";
+ find.processOptions(getArgs(args));
+ Expression expression = find.getRootExpression();
+ assertEquals(expected, expression.toString());
+ }
+
+ // check -print is handled correctly
+ @Test(timeout = 1000)
+ public void processOptionsPrint() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+ String args = "path -print";
+ String expected = "Print(;)";
+ find.processOptions(getArgs(args));
+ Expression expression = find.getRootExpression();
+ assertEquals(expected, expression.toString());
+ }
+
+ // check -print0 is handled correctly
+ @Test(timeout = 1000)
+ public void processOptionsPrint0() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+ String args = "path -print0";
+ String expected = "Print0-Print(;)";
+ find.processOptions(getArgs(args));
+ Expression expression = find.getRootExpression();
+ assertEquals(expected, expression.toString());
+ }
+
+ // check an implicit and is handled correctly
+ @Test(timeout = 1000)
+ public void processOptionsNoop() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+
+ String args = "path -name one -name two -print";
+ String expected = "And(;And(;Name(one;),Name(two;)),Print(;))";
+ find.processOptions(getArgs(args));
+ Expression expression = find.getRootExpression();
+ assertEquals(expected, expression.toString());
+ }
+
+ // check -a is handled correctly
+ @Test(timeout = 1000)
+ public void processOptionsA() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+
+ String args = "path -name one -a -name two -a -print";
+ String expected = "And(;And(;Name(one;),Name(two;)),Print(;))";
+ find.processOptions(getArgs(args));
+ Expression expression = find.getRootExpression();
+ assertEquals(expected, expression.toString());
+ }
+
+ // check -and is handled correctly
+ @Test(timeout = 1000)
+ public void processOptionsAnd() throws IOException {
+ Find find = new Find();
+ find.setConf(conf);
+
+ String args = "path -name one -and -name two -and -print";
+ String expected = "And(;And(;Name(one;),Name(two;)),Print(;))";
+ find.processOptions(getArgs(args));
+ Expression expression = find.getRootExpression();
+ assertEquals(expected, expression.toString());
+ }
+
+ // check expressions are called in the correct order
+ @Test(timeout = 1000)
+ public void processArguments() throws IOException {
+ LinkedList<PathData> items = createDirectories();
+
+ Find find = new Find();
+ find.setConf(conf);
+ PrintStream out = mock(PrintStream.class);
+ find.getOptions().setOut(out);
+ PrintStream err = mock(PrintStream.class);
+ find.getOptions().setErr(err);
+ Expression expr = mock(Expression.class);
+ when(expr.apply((PathData) any(), anyInt())).thenReturn(Result.PASS);
+ FileStatusChecker fsCheck = mock(FileStatusChecker.class);
+ Expression test = new TestExpression(expr, fsCheck);
+ find.setRootExpression(test);
+ find.processArguments(items);
+
+ InOrder inOrder = inOrder(expr);
+ inOrder.verify(expr).setOptions(find.getOptions());
+ inOrder.verify(expr).prepare();
+ inOrder.verify(expr).apply(item1, 0);
+ inOrder.verify(expr).apply(item1a, 1);
+ inOrder.verify(expr).apply(item1aa, 2);
+ inOrder.verify(expr).apply(item1b, 1);
+ inOrder.verify(expr).apply(item2, 0);
+ inOrder.verify(expr).apply(item3, 0);
+ inOrder.verify(expr).apply(item4, 0);
+ inOrder.verify(expr).apply(item5, 0);
+ inOrder.verify(expr).apply(item5a, 1);
+ inOrder.verify(expr).apply(item5b, 1);
+ inOrder.verify(expr).apply(item5c, 1);
+ inOrder.verify(expr).apply(item5ca, 2);
+ inOrder.verify(expr).apply(item5d, 1);
+ inOrder.verify(expr).apply(item5e, 1);
+ inOrder.verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+
+ InOrder inOrderFsCheck = inOrder(fsCheck);
+ inOrderFsCheck.verify(fsCheck).check(item1.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1aa.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item2.stat);
+ inOrderFsCheck.verify(fsCheck).check(item3.stat);
+ inOrderFsCheck.verify(fsCheck).check(item4.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5ca.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5d.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5e.stat);
+ verifyNoMoreInteractions(fsCheck);
+
+ verifyNoMoreInteractions(out);
+ verifyNoMoreInteractions(err);
+ }
+
+ // check that directories are descended correctly when -depth is specified
+ @Test(timeout = 1000)
+ public void processArgumentsDepthFirst() throws IOException {
+ LinkedList<PathData> items = createDirectories();
+
+ Find find = new Find();
+ find.getOptions().setDepthFirst(true);
+ find.setConf(conf);
+ PrintStream out = mock(PrintStream.class);
+ find.getOptions().setOut(out);
+ PrintStream err = mock(PrintStream.class);
+ find.getOptions().setErr(err);
+ Expression expr = mock(Expression.class);
+ when(expr.apply((PathData) any(), anyInt())).thenReturn(Result.PASS);
+ FileStatusChecker fsCheck = mock(FileStatusChecker.class);
+ Expression test = new TestExpression(expr, fsCheck);
+ find.setRootExpression(test);
+ find.processArguments(items);
+
+ InOrder inOrder = inOrder(expr);
+ inOrder.verify(expr).setOptions(find.getOptions());
+ inOrder.verify(expr).prepare();
+ inOrder.verify(expr).apply(item1aa, 2);
+ inOrder.verify(expr).apply(item1a, 1);
+ inOrder.verify(expr).apply(item1b, 1);
+ inOrder.verify(expr).apply(item1, 0);
+ inOrder.verify(expr).apply(item2, 0);
+ inOrder.verify(expr).apply(item3, 0);
+ inOrder.verify(expr).apply(item4, 0);
+ inOrder.verify(expr).apply(item5a, 1);
+ inOrder.verify(expr).apply(item5b, 1);
+ inOrder.verify(expr).apply(item5ca, 2);
+ inOrder.verify(expr).apply(item5c, 1);
+ inOrder.verify(expr).apply(item5d, 1);
+ inOrder.verify(expr).apply(item5e, 1);
+ inOrder.verify(expr).apply(item5, 0);
+ inOrder.verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+
+ InOrder inOrderFsCheck = inOrder(fsCheck);
+ inOrderFsCheck.verify(fsCheck).check(item1aa.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1.stat);
+ inOrderFsCheck.verify(fsCheck).check(item2.stat);
+ inOrderFsCheck.verify(fsCheck).check(item3.stat);
+ inOrderFsCheck.verify(fsCheck).check(item4.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5ca.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5d.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5e.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5.stat);
+ verifyNoMoreInteractions(fsCheck);
+
+ verifyNoMoreInteractions(out);
+ verifyNoMoreInteractions(err);
+ }
+
+ // check symlinks given as path arguments are processed correctly with the
+ // follow arg option set
+ @Test(timeout = 1000)
+ public void processArgumentsOptionFollowArg() throws IOException {
+ LinkedList<PathData> items = createDirectories();
+
+ Find find = new Find();
+ find.getOptions().setFollowArgLink(true);
+ find.setConf(conf);
+ PrintStream out = mock(PrintStream.class);
+ find.getOptions().setOut(out);
+ PrintStream err = mock(PrintStream.class);
+ find.getOptions().setErr(err);
+ Expression expr = mock(Expression.class);
+ when(expr.apply((PathData) any(), anyInt())).thenReturn(Result.PASS);
+ FileStatusChecker fsCheck = mock(FileStatusChecker.class);
+ Expression test = new TestExpression(expr, fsCheck);
+ find.setRootExpression(test);
+ find.processArguments(items);
+
+ InOrder inOrder = inOrder(expr);
+ inOrder.verify(expr).setOptions(find.getOptions());
+ inOrder.verify(expr).prepare();
+ inOrder.verify(expr).apply(item1, 0);
+ inOrder.verify(expr).apply(item1a, 1);
+ inOrder.verify(expr).apply(item1aa, 2);
+ inOrder.verify(expr).apply(item1b, 1);
+ inOrder.verify(expr).apply(item2, 0);
+ inOrder.verify(expr).apply(item3, 0);
+ inOrder.verify(expr).apply(item4, 0);
+ inOrder.verify(expr).apply(item5, 0);
+ inOrder.verify(expr).apply(item5a, 1);
+ inOrder.verify(expr).apply(item5b, 1);
+ inOrder.verify(expr).apply(item5c, 1);
+ inOrder.verify(expr).apply(item5ca, 2);
+ inOrder.verify(expr).apply(item5d, 1);
+ inOrder.verify(expr).apply(item5e, 1);
+ inOrder.verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+
+ InOrder inOrderFsCheck = inOrder(fsCheck);
+ inOrderFsCheck.verify(fsCheck).check(item1.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1aa.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item2.stat);
+ inOrderFsCheck.verify(fsCheck, times(2)).check(item3.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5ca.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5d.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5e.stat);
+ verifyNoMoreInteractions(fsCheck);
+
+ verifyNoMoreInteractions(out);
+ verifyNoMoreInteractions(err);
+ }
+
+ // check symlinks given as path arguments are processed correctly with the
+ // follow option
+ @Test(timeout = 1000)
+ public void processArgumentsOptionFollow() throws IOException {
+ LinkedList<PathData> items = createDirectories();
+
+ Find find = new Find();
+ find.getOptions().setFollowLink(true);
+ find.setConf(conf);
+ PrintStream out = mock(PrintStream.class);
+ find.getOptions().setOut(out);
+ PrintStream err = mock(PrintStream.class);
+ find.getOptions().setErr(err);
+ Expression expr = mock(Expression.class);
+ when(expr.apply((PathData) any(), anyInt())).thenReturn(Result.PASS);
+ FileStatusChecker fsCheck = mock(FileStatusChecker.class);
+ Expression test = new TestExpression(expr, fsCheck);
+ find.setRootExpression(test);
+ find.processArguments(items);
+
+ InOrder inOrder = inOrder(expr);
+ inOrder.verify(expr).setOptions(find.getOptions());
+ inOrder.verify(expr).prepare();
+ inOrder.verify(expr).apply(item1, 0);
+ inOrder.verify(expr).apply(item1a, 1);
+ inOrder.verify(expr).apply(item1aa, 2);
+ inOrder.verify(expr).apply(item1b, 1);
+ inOrder.verify(expr).apply(item2, 0);
+ inOrder.verify(expr).apply(item3, 0);
+ inOrder.verify(expr).apply(item4, 0);
+ inOrder.verify(expr).apply(item5, 0);
+ inOrder.verify(expr).apply(item5a, 1);
+ inOrder.verify(expr).apply(item5b, 1); // triggers infinite loop message
+ inOrder.verify(expr).apply(item5c, 1);
+ inOrder.verify(expr).apply(item5ca, 2);
+ inOrder.verify(expr).apply(item5d, 1);
+ inOrder.verify(expr).apply(item5ca, 2); // following item5d symlink
+ inOrder.verify(expr).apply(item5e, 1);
+ inOrder.verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+
+ InOrder inOrderFsCheck = inOrder(fsCheck);
+ inOrderFsCheck.verify(fsCheck).check(item1.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1aa.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item2.stat);
+ inOrderFsCheck.verify(fsCheck, times(2)).check(item3.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5ca.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck, times(2)).check(item5ca.stat);
+ verifyNoMoreInteractions(fsCheck);
+
+ verifyNoMoreInteractions(out);
+ verify(err).println(
+ "Infinite loop ignored: " + item5b.toString() + " -> "
+ + item5.toString());
+ verifyNoMoreInteractions(err);
+ }
+
+ // check minimum depth is handledfollowLink
+ @Test(timeout = 1000)
+ public void processArgumentsMinDepth() throws IOException {
+ LinkedList<PathData> items = createDirectories();
+
+ Find find = new Find();
+ find.getOptions().setMinDepth(1);
+ find.setConf(conf);
+ PrintStream out = mock(PrintStream.class);
+ find.getOptions().setOut(out);
+ PrintStream err = mock(PrintStream.class);
+ find.getOptions().setErr(err);
+ Expression expr = mock(Expression.class);
+ when(expr.apply((PathData) any(), anyInt())).thenReturn(Result.PASS);
+ FileStatusChecker fsCheck = mock(FileStatusChecker.class);
+ Expression test = new TestExpression(expr, fsCheck);
+ find.setRootExpression(test);
+ find.processArguments(items);
+
+ InOrder inOrder = inOrder(expr);
+ inOrder.verify(expr).setOptions(find.getOptions());
+ inOrder.verify(expr).prepare();
+ inOrder.verify(expr).apply(item1a, 1);
+ inOrder.verify(expr).apply(item1aa, 2);
+ inOrder.verify(expr).apply(item1b, 1);
+ inOrder.verify(expr).apply(item5a, 1);
+ inOrder.verify(expr).apply(item5b, 1);
+ inOrder.verify(expr).apply(item5c, 1);
+ inOrder.verify(expr).apply(item5ca, 2);
+ inOrder.verify(expr).apply(item5d, 1);
+ inOrder.verify(expr).apply(item5e, 1);
+ inOrder.verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+
+ InOrder inOrderFsCheck = inOrder(fsCheck);
+ inOrderFsCheck.verify(fsCheck).check(item1a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1aa.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5ca.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5d.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5e.stat);
+ verifyNoMoreInteractions(fsCheck);
+
+ verifyNoMoreInteractions(out);
+ verifyNoMoreInteractions(err);
+ }
+
+ // check maximum depth is handled
+ @Test(timeout = 1000)
+ public void processArgumentsMaxDepth() throws IOException {
+ LinkedList<PathData> items = createDirectories();
+
+ Find find = new Find();
+ find.getOptions().setMaxDepth(1);
+ find.setConf(conf);
+ PrintStream out = mock(PrintStream.class);
+ find.getOptions().setOut(out);
+ PrintStream err = mock(PrintStream.class);
+ find.getOptions().setErr(err);
+ Expression expr = mock(Expression.class);
+ when(expr.apply((PathData) any(), anyInt())).thenReturn(Result.PASS);
+ FileStatusChecker fsCheck = mock(FileStatusChecker.class);
+ Expression test = new TestExpression(expr, fsCheck);
+ find.setRootExpression(test);
+ find.processArguments(items);
+
+ InOrder inOrder = inOrder(expr);
+ inOrder.verify(expr).setOptions(find.getOptions());
+ inOrder.verify(expr).prepare();
+ inOrder.verify(expr).apply(item1, 0);
+ inOrder.verify(expr).apply(item1a, 1);
+ inOrder.verify(expr).apply(item1b, 1);
+ inOrder.verify(expr).apply(item2, 0);
+ inOrder.verify(expr).apply(item3, 0);
+ inOrder.verify(expr).apply(item4, 0);
+ inOrder.verify(expr).apply(item5, 0);
+ inOrder.verify(expr).apply(item5a, 1);
+ inOrder.verify(expr).apply(item5b, 1);
+ inOrder.verify(expr).apply(item5c, 1);
+ inOrder.verify(expr).apply(item5d, 1);
+ inOrder.verify(expr).apply(item5e, 1);
+ inOrder.verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+
+ InOrder inOrderFsCheck = inOrder(fsCheck);
+ inOrderFsCheck.verify(fsCheck).check(item1.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item2.stat);
+ inOrderFsCheck.verify(fsCheck).check(item3.stat);
+ inOrderFsCheck.verify(fsCheck).check(item4.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5d.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5e.stat);
+ verifyNoMoreInteractions(fsCheck);
+
+ verifyNoMoreInteractions(out);
+ verifyNoMoreInteractions(err);
+ }
+
+ // check min depth is handled when -depth is specified
+ @Test(timeout = 1000)
+ public void processArgumentsDepthFirstMinDepth() throws IOException {
+ LinkedList<PathData> items = createDirectories();
+
+ Find find = new Find();
+ find.getOptions().setDepthFirst(true);
+ find.getOptions().setMinDepth(1);
+ find.setConf(conf);
+ PrintStream out = mock(PrintStream.class);
+ find.getOptions().setOut(out);
+ PrintStream err = mock(PrintStream.class);
+ find.getOptions().setErr(err);
+ Expression expr = mock(Expression.class);
+ when(expr.apply((PathData) any(), anyInt())).thenReturn(Result.PASS);
+ FileStatusChecker fsCheck = mock(FileStatusChecker.class);
+ Expression test = new TestExpression(expr, fsCheck);
+ find.setRootExpression(test);
+ find.processArguments(items);
+
+ InOrder inOrder = inOrder(expr);
+ inOrder.verify(expr).setOptions(find.getOptions());
+ inOrder.verify(expr).prepare();
+ inOrder.verify(expr).apply(item1aa, 2);
+ inOrder.verify(expr).apply(item1a, 1);
+ inOrder.verify(expr).apply(item1b, 1);
+ inOrder.verify(expr).apply(item5a, 1);
+ inOrder.verify(expr).apply(item5b, 1);
+ inOrder.verify(expr).apply(item5ca, 2);
+ inOrder.verify(expr).apply(item5c, 1);
+ inOrder.verify(expr).apply(item5d, 1);
+ inOrder.verify(expr).apply(item5e, 1);
+ inOrder.verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+
+ InOrder inOrderFsCheck = inOrder(fsCheck);
+ inOrderFsCheck.verify(fsCheck).check(item1aa.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5ca.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5d.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5e.stat);
+ verifyNoMoreInteractions(fsCheck);
+
+ verifyNoMoreInteractions(out);
+ verifyNoMoreInteractions(err);
+ }
+
+ // check max depth is handled when -depth is specified
+ @Test(timeout = 1000)
+ public void processArgumentsDepthFirstMaxDepth() throws IOException {
+ LinkedList<PathData> items = createDirectories();
+
+ Find find = new Find();
+ find.getOptions().setDepthFirst(true);
+ find.getOptions().setMaxDepth(1);
+ find.setConf(conf);
+ PrintStream out = mock(PrintStream.class);
+ find.getOptions().setOut(out);
+ PrintStream err = mock(PrintStream.class);
+ find.getOptions().setErr(err);
+ Expression expr = mock(Expression.class);
+ when(expr.apply((PathData) any(), anyInt())).thenReturn(Result.PASS);
+ FileStatusChecker fsCheck = mock(FileStatusChecker.class);
+ Expression test = new TestExpression(expr, fsCheck);
+ find.setRootExpression(test);
+ find.processArguments(items);
+
+ InOrder inOrder = inOrder(expr);
+ inOrder.verify(expr).setOptions(find.getOptions());
+ inOrder.verify(expr).prepare();
+ inOrder.verify(expr).apply(item1a, 1);
+ inOrder.verify(expr).apply(item1b, 1);
+ inOrder.verify(expr).apply(item1, 0);
+ inOrder.verify(expr).apply(item2, 0);
+ inOrder.verify(expr).apply(item3, 0);
+ inOrder.verify(expr).apply(item4, 0);
+ inOrder.verify(expr).apply(item5a, 1);
+ inOrder.verify(expr).apply(item5b, 1);
+ inOrder.verify(expr).apply(item5c, 1);
+ inOrder.verify(expr).apply(item5d, 1);
+ inOrder.verify(expr).apply(item5e, 1);
+ inOrder.verify(expr).apply(item5, 0);
+ inOrder.verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+
+ InOrder inOrderFsCheck = inOrder(fsCheck);
+ inOrderFsCheck.verify(fsCheck).check(item1a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1.stat);
+ inOrderFsCheck.verify(fsCheck).check(item2.stat);
+ inOrderFsCheck.verify(fsCheck).check(item3.stat);
+ inOrderFsCheck.verify(fsCheck).check(item4.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5d.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5e.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5.stat);
+ verifyNoMoreInteractions(fsCheck);
+
+ verifyNoMoreInteractions(out);
+ verifyNoMoreInteractions(err);
+ }
+
+ // check expressions are called in the correct order
+ @Test(timeout = 1000)
+ public void processArgumentsNoDescend() throws IOException {
+ LinkedList<PathData> items = createDirectories();
+
+ Find find = new Find();
+ find.setConf(conf);
+ PrintStream out = mock(PrintStream.class);
+ find.getOptions().setOut(out);
+ PrintStream err = mock(PrintStream.class);
+ find.getOptions().setErr(err);
+ Expression expr = mock(Expression.class);
+ when(expr.apply((PathData) any(), anyInt())).thenReturn(Result.PASS);
+ when(expr.apply(eq(item1a), anyInt())).thenReturn(Result.STOP);
+ FileStatusChecker fsCheck = mock(FileStatusChecker.class);
+ Expression test = new TestExpression(expr, fsCheck);
+ find.setRootExpression(test);
+ find.processArguments(items);
+
+ InOrder inOrder = inOrder(expr);
+ inOrder.verify(expr).setOptions(find.getOptions());
+ inOrder.verify(expr).prepare();
+ inOrder.verify(expr).apply(item1, 0);
+ inOrder.verify(expr).apply(item1a, 1);
+ inOrder.verify(expr).apply(item1b, 1);
+ inOrder.verify(expr).apply(item2, 0);
+ inOrder.verify(expr).apply(item3, 0);
+ inOrder.verify(expr).apply(item4, 0);
+ inOrder.verify(expr).apply(item5, 0);
+ inOrder.verify(expr).apply(item5a, 1);
+ inOrder.verify(expr).apply(item5b, 1);
+ inOrder.verify(expr).apply(item5c, 1);
+ inOrder.verify(expr).apply(item5ca, 2);
+ inOrder.verify(expr).apply(item5d, 1);
+ inOrder.verify(expr).apply(item5e, 1);
+ inOrder.verify(expr).finish();
+ verifyNoMoreInteractions(expr);
+
+ InOrder inOrderFsCheck = inOrder(fsCheck);
+ inOrderFsCheck.verify(fsCheck).check(item1.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item1b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item2.stat);
+ inOrderFsCheck.verify(fsCheck).check(item3.stat);
+ inOrderFsCheck.verify(fsCheck).check(item4.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5a.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5b.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5c.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5ca.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5d.stat);
+ inOrderFsCheck.verify(fsCheck).check(item5e.stat);
+ verifyNoMoreInteractions(fsCheck);
+
+ verifyNoMoreInteractions(out);
+ verifyNoMoreInteractions(err);
+ }
+
+ private interface FileStatusChecker {
+ public void check(FileStatus fileStatus);
+ }
+
+ private class TestExpression extends BaseExpression implements Expression {
+ private Expression expr;
+ private FileStatusChecker checker;
+ public TestExpression(Expression expr, FileStatusChecker checker) {
+ this.expr = expr;
+ this.checker = checker;
+ }
+ @Override
+ public Result apply(PathData item, int depth) throws IOException {
+ FileStatus fileStatus = getFileStatus(item, depth);
+ checker.check(fileStatus);
+ return expr.apply(item, depth);
+ }
+ @Override
+ public void setOptions(FindOptions options) throws IOException {
+ super.setOptions(options);
+ expr.setOptions(options);
+ }
+ @Override
+ public void prepare() throws IOException {
+ expr.prepare();
+ }
+ @Override
+ public void finish() throws IOException {
+ expr.finish();
+ }
+ }
+
+ // creates a directory structure for traversal
+ // item1 (directory)
+ // \- item1a (directory)
+ // \- item1aa (file)
+ // \- item1b (file)
+ // item2 (directory)
+ // item3 (file)
+ // item4 (link) -> item3
+ // item5 (directory)
+ // \- item5a (link) -> item1b
+ // \- item5b (link) -> item5 (infinite loop)
+ // \- item5c (directory)
+ // \- item5ca (file)
+ // \- item5d (link) -> item5c
+ // \- item5e (link) -> item5c/item5ca
+ private PathData item1 = null;
+ private PathData item1a = null;
+ private PathData item1aa = null;
+ private PathData item1b = null;
+ private PathData item2 = null;
+ private PathData item3 = null;
+ private PathData item4 = null;
+ private PathData item5 = null;
+ private PathData item5a = null;
+ private PathData item5b = null;
+ private PathData item5c = null;
+ private PathData item5ca = null;
+ private PathData item5d = null;
+ private PathData item5e = null;
+
+ private LinkedList<PathData> createDirectories() throws IOException {
+ item1 = createPathData("item1");
+ item1a = createPathData("item1/item1a");
+ item1aa = createPathData("item1/item1a/item1aa");
+ item1b = createPathData("item1/item1b");
+ item2 = createPathData("item2");
+ item3 = createPathData("item3");
+ item4 = createPathData("item4");
+ item5 = createPathData("item5");
+ item5a = createPathData("item5/item5a");
+ item5b = createPathData("item5/item5b");
+ item5c = createPathData("item5/item5c");
+ item5ca = createPathData("item5/item5c/item5ca");
+ item5d = createPathData("item5/item5d");
+ item5e = createPathData("item5/item5e");
+
+ LinkedList<PathData> args = new LinkedList<PathData>();
+
+ when(item1.stat.isDirectory()).thenReturn(true);
+ when(item1a.stat.isDirectory()).thenReturn(true);
+ when(item1aa.stat.isDirectory()).thenReturn(false);
+ when(item1b.stat.isDirectory()).thenReturn(false);
+ when(item2.stat.isDirectory()).thenReturn(true);
+ when(item3.stat.isDirectory()).thenReturn(false);
+ when(item4.stat.isDirectory()).thenReturn(false);
+ when(item5.stat.isDirectory()).thenReturn(true);
+ when(item5a.stat.isDirectory()).thenReturn(false);
+ when(item5b.stat.isDirectory()).thenReturn(false);
+ when(item5c.stat.isDirectory()).thenReturn(true);
+ when(item5ca.stat.isDirectory()).thenReturn(false);
+ when(item5d.stat.isDirectory()).thenReturn(false);
+ when(item5e.stat.isDirectory()).thenReturn(false);
+
+ when(mockFs.listStatus(eq(item1.path))).thenReturn(
+ new FileStatus[] { item1a.stat, item1b.stat });
+ when(mockFs.listStatus(eq(item1a.path))).thenReturn(
+ new FileStatus[] { item1aa.stat });
+ when(mockFs.listStatus(eq(item2.path))).thenReturn(new FileStatus[0]);
+ when(mockFs.listStatus(eq(item5.path))).thenReturn(
+ new FileStatus[] { item5a.stat, item5b.stat, item5c.stat, item5d.stat,
+ item5e.stat });
+ when(mockFs.listStatus(eq(item5c.path))).thenReturn(
+ new FileStatus[] { item5ca.stat });
+
+ when(item1.stat.isSymlink()).thenReturn(false);
+ when(item1a.stat.isSymlink()).thenReturn(false);
+ when(item1aa.stat.isSymlink()).thenReturn(false);
+ when(item1b.stat.isSymlink()).thenReturn(false);
+ when(item2.stat.isSymlink()).thenReturn(false);
+ when(item3.stat.isSymlink()).thenReturn(false);
+ when(item4.stat.isSymlink()).thenReturn(true);
+ when(item5.stat.isSymlink()).thenReturn(false);
+ when(item5a.stat.isSymlink()).thenReturn(true);
+ when(item5b.stat.isSymlink()).thenReturn(true);
+ when(item5d.stat.isSymlink()).thenReturn(true);
+ when(item5e.stat.isSymlink()).thenReturn(true);
+
+ when(item4.stat.getSymlink()).thenReturn(item3.path);
+ when(item5a.stat.getSymlink()).thenReturn(item1b.path);
+ when(item5b.stat.getSymlink()).thenReturn(item5.path);
+ when(item5d.stat.getSymlink()).thenReturn(item5c.path);
+ when(item5e.stat.getSymlink()).thenReturn(item5ca.path);
+
+ args.add(item1);
+ args.add(item2);
+ args.add(item3);
+ args.add(item4);
+ args.add(item5);
+
+ return args;
+ }
+
+ private PathData createPathData(String name) throws IOException {
+ Path path = new Path(name);
+ FileStatus fstat = mock(FileStatus.class);
+ when(fstat.getPath()).thenReturn(path);
+ when(fstat.toString()).thenReturn("fileStatus:" + name);
+
+ when(mockFs.getFileStatus(eq(path))).thenReturn(fstat);
+ PathData item = new PathData(path.toString(), conf);
+ return item;
+ }
+
+ private LinkedList<String> getArgs(String cmd) {
+ return new LinkedList<String>(Arrays.asList(cmd.split(" ")));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestHelper.java
new file mode 100644
index 0000000..d4866b5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestHelper.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+
+/** Helper methods for the find expression unit tests. */
+class TestHelper {
+ /** Adds an argument string to an expression */
+ static void addArgument(Expression expr, String arg) {
+ expr.addArguments(new LinkedList<String>(Collections.singletonList(arg)));
+ }
+
+ /** Converts a command string into a list of arguments. */
+ static LinkedList<String> getArgs(String cmd) {
+ return new LinkedList<String>(Arrays.asList(cmd.split(" ")));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestIname.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestIname.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestIname.java
new file mode 100644
index 0000000..6e42fce
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestIname.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import static org.junit.Assert.*;
+import static org.apache.hadoop.fs.shell.find.TestHelper.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.shell.PathData;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestIname {
+ private FileSystem mockFs;
+ private Name.Iname name;
+
+ @Before
+ public void resetMock() throws IOException {
+ mockFs = MockFileSystem.setup();
+ }
+
+ private void setup(String arg) throws IOException {
+ name = new Name.Iname();
+ addArgument(name, arg);
+ name.setOptions(new FindOptions());
+ name.prepare();
+ }
+
+ // test a matching name (same case)
+ @Test(timeout = 1000)
+ public void applyMatch() throws IOException {
+ setup("name");
+ PathData item = new PathData("/directory/path/name", mockFs.getConf());
+ assertEquals(Result.PASS, name.apply(item, -1));
+ }
+
+ // test a non-matching name
+ @Test(timeout = 1000)
+ public void applyNotMatch() throws IOException {
+ setup("name");
+ PathData item = new PathData("/directory/path/notname", mockFs.getConf());
+ assertEquals(Result.FAIL, name.apply(item, -1));
+ }
+
+ // test a matching name (different case)
+ @Test(timeout = 1000)
+ public void applyMixedCase() throws IOException {
+ setup("name");
+ PathData item = new PathData("/directory/path/NaMe", mockFs.getConf());
+ assertEquals(Result.PASS, name.apply(item, -1));
+ }
+
+ // test a matching glob pattern (same case)
+ @Test(timeout = 1000)
+ public void applyGlob() throws IOException {
+ setup("n*e");
+ PathData item = new PathData("/directory/path/name", mockFs.getConf());
+ assertEquals(Result.PASS, name.apply(item, -1));
+ }
+
+ // test a matching glob pattern (different case)
+ @Test(timeout = 1000)
+ public void applyGlobMixedCase() throws IOException {
+ setup("n*e");
+ PathData item = new PathData("/directory/path/NaMe", mockFs.getConf());
+ assertEquals(Result.PASS, name.apply(item, -1));
+ }
+
+ // test a non-matching glob pattern
+ @Test(timeout = 1000)
+ public void applyGlobNotMatch() throws IOException {
+ setup("n*e");
+ PathData item = new PathData("/directory/path/notmatch", mockFs.getConf());
+ assertEquals(Result.FAIL, name.apply(item, -1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestName.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestName.java
new file mode 100644
index 0000000..2c77fe1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestName.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import static org.junit.Assert.*;
+import static org.apache.hadoop.fs.shell.find.TestHelper.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.shell.PathData;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestName {
+ private FileSystem mockFs;
+ private Name name;
+
+ @Before
+ public void resetMock() throws IOException {
+ mockFs = MockFileSystem.setup();
+ }
+
+ private void setup(String arg) throws IOException {
+ name = new Name();
+ addArgument(name, arg);
+ name.setOptions(new FindOptions());
+ name.prepare();
+ }
+
+ // test a matching name
+ @Test(timeout = 1000)
+ public void applyMatch() throws IOException {
+ setup("name");
+ PathData item = new PathData("/directory/path/name", mockFs.getConf());
+ assertEquals(Result.PASS, name.apply(item, -1));
+ }
+
+ // test a non-matching name
+ @Test(timeout = 1000)
+ public void applyNotMatch() throws IOException {
+ setup("name");
+ PathData item = new PathData("/directory/path/notname", mockFs.getConf());
+ assertEquals(Result.FAIL, name.apply(item, -1));
+ }
+
+ // test a different case name
+ @Test(timeout = 1000)
+ public void applyMixedCase() throws IOException {
+ setup("name");
+ PathData item = new PathData("/directory/path/NaMe", mockFs.getConf());
+ assertEquals(Result.FAIL, name.apply(item, -1));
+ }
+
+ // test a matching glob pattern
+ @Test(timeout = 1000)
+ public void applyGlob() throws IOException {
+ setup("n*e");
+ PathData item = new PathData("/directory/path/name", mockFs.getConf());
+ assertEquals(Result.PASS, name.apply(item, -1));
+ }
+
+ // test a glob pattern with different case
+ @Test(timeout = 1000)
+ public void applyGlobMixedCase() throws IOException {
+ setup("n*e");
+ PathData item = new PathData("/directory/path/NaMe", mockFs.getConf());
+ assertEquals(Result.FAIL, name.apply(item, -1));
+ }
+
+ // test a non-matching glob pattern
+ @Test(timeout = 1000)
+ public void applyGlobNotMatch() throws IOException {
+ setup("n*e");
+ PathData item = new PathData("/directory/path/notmatch", mockFs.getConf());
+ assertEquals(Result.FAIL, name.apply(item, -1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestPrint.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestPrint.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestPrint.java
new file mode 100644
index 0000000..2d27665
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestPrint.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.shell.PathData;
+import org.junit.Test;
+
+import java.io.PrintStream;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.junit.Before;
+
+public class TestPrint {
+ private FileSystem mockFs;
+
+ @Before
+ public void resetMock() throws IOException {
+ mockFs = MockFileSystem.setup();
+ }
+
+ // test the full path is printed to stdout
+ @Test(timeout = 1000)
+ public void testPrint() throws IOException {
+ Print print = new Print();
+ PrintStream out = mock(PrintStream.class);
+ FindOptions options = new FindOptions();
+ options.setOut(out);
+ print.setOptions(options);
+
+ String filename = "/one/two/test";
+ PathData item = new PathData(filename, mockFs.getConf());
+ assertEquals(Result.PASS, print.apply(item, -1));
+ verify(out).print(filename + '\n');
+ verifyNoMoreInteractions(out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestPrint0.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestPrint0.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestPrint0.java
new file mode 100644
index 0000000..3b89438
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestPrint0.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.shell.PathData;
+import org.junit.Test;
+
+import java.io.PrintStream;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.junit.Before;
+
+public class TestPrint0 {
+ private FileSystem mockFs;
+
+ @Before
+ public void resetMock() throws IOException {
+ mockFs = MockFileSystem.setup();
+ }
+
+ // test the full path is printed to stdout with a '\0'
+ @Test(timeout = 1000)
+ public void testPrint() throws IOException {
+ Print.Print0 print = new Print.Print0();
+ PrintStream out = mock(PrintStream.class);
+ FindOptions options = new FindOptions();
+ options.setOut(out);
+ print.setOptions(options);
+
+ String filename = "/one/two/test";
+ PathData item = new PathData(filename, mockFs.getConf());
+ assertEquals(Result.PASS, print.apply(item, -1));
+ verify(out).print(filename + '\0');
+ verifyNoMoreInteractions(out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestResult.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestResult.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestResult.java
new file mode 100644
index 0000000..1139220
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/find/TestResult.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell.find;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class TestResult {
+
+ // test the PASS value
+ @Test(timeout = 1000)
+ public void testPass() {
+ Result result = Result.PASS;
+ assertTrue(result.isPass());
+ assertTrue(result.isDescend());
+ }
+
+ // test the FAIL value
+ @Test(timeout = 1000)
+ public void testFail() {
+ Result result = Result.FAIL;
+ assertFalse(result.isPass());
+ assertTrue(result.isDescend());
+ }
+
+ // test the STOP value
+ @Test(timeout = 1000)
+ public void testStop() {
+ Result result = Result.STOP;
+ assertTrue(result.isPass());
+ assertFalse(result.isDescend());
+ }
+
+ // test combine method with two PASSes
+ @Test(timeout = 1000)
+ public void combinePassPass() {
+ Result result = Result.PASS.combine(Result.PASS);
+ assertTrue(result.isPass());
+ assertTrue(result.isDescend());
+ }
+
+ // test the combine method with a PASS and a FAIL
+ @Test(timeout = 1000)
+ public void combinePassFail() {
+ Result result = Result.PASS.combine(Result.FAIL);
+ assertFalse(result.isPass());
+ assertTrue(result.isDescend());
+ }
+
+ // test the combine method with a FAIL and a PASS
+ @Test(timeout = 1000)
+ public void combineFailPass() {
+ Result result = Result.FAIL.combine(Result.PASS);
+ assertFalse(result.isPass());
+ assertTrue(result.isDescend());
+ }
+
+ // test the combine method with two FAILs
+ @Test(timeout = 1000)
+ public void combineFailFail() {
+ Result result = Result.FAIL.combine(Result.FAIL);
+ assertFalse(result.isPass());
+ assertTrue(result.isDescend());
+ }
+
+ // test the combine method with a PASS and STOP
+ @Test(timeout = 1000)
+ public void combinePassStop() {
+ Result result = Result.PASS.combine(Result.STOP);
+ assertTrue(result.isPass());
+ assertFalse(result.isDescend());
+ }
+
+ // test the combine method with a STOP and FAIL
+ @Test(timeout = 1000)
+ public void combineStopFail() {
+ Result result = Result.STOP.combine(Result.FAIL);
+ assertFalse(result.isPass());
+ assertFalse(result.isDescend());
+ }
+
+ // test the combine method with a STOP and a PASS
+ @Test(timeout = 1000)
+ public void combineStopPass() {
+ Result result = Result.STOP.combine(Result.PASS);
+ assertTrue(result.isPass());
+ assertFalse(result.isDescend());
+ }
+
+ // test the combine method with a FAIL and a STOP
+ @Test(timeout = 1000)
+ public void combineFailStop() {
+ Result result = Result.FAIL.combine(Result.STOP);
+ assertFalse(result.isPass());
+ assertFalse(result.isDescend());
+ }
+
+ // test the negation of PASS
+ @Test(timeout = 1000)
+ public void negatePass() {
+ Result result = Result.PASS.negate();
+ assertFalse(result.isPass());
+ assertTrue(result.isDescend());
+ }
+
+ // test the negation of FAIL
+ @Test(timeout = 1000)
+ public void negateFail() {
+ Result result = Result.FAIL.negate();
+ assertTrue(result.isPass());
+ assertTrue(result.isDescend());
+ }
+
+ // test the negation of STOP
+ @Test(timeout = 1000)
+ public void negateStop() {
+ Result result = Result.STOP.negate();
+ assertFalse(result.isPass());
+ assertFalse(result.isDescend());
+ }
+
+ // test equals with two PASSes
+ @Test(timeout = 1000)
+ public void equalsPass() {
+ Result one = Result.PASS;
+ Result two = Result.PASS.combine(Result.PASS);
+ assertEquals(one, two);
+ }
+
+ // test equals with two FAILs
+ @Test(timeout = 1000)
+ public void equalsFail() {
+ Result one = Result.FAIL;
+ Result two = Result.FAIL.combine(Result.FAIL);
+ assertEquals(one, two);
+ }
+
+ // test equals with two STOPS
+ @Test(timeout = 1000)
+ public void equalsStop() {
+ Result one = Result.STOP;
+ Result two = Result.STOP.combine(Result.STOP);
+ assertEquals(one, two);
+ }
+
+ // test all combinations of not equals
+ @Test(timeout = 1000)
+ public void notEquals() {
+ assertFalse(Result.PASS.equals(Result.FAIL));
+ assertFalse(Result.PASS.equals(Result.STOP));
+ assertFalse(Result.FAIL.equals(Result.PASS));
+ assertFalse(Result.FAIL.equals(Result.STOP));
+ assertFalse(Result.STOP.equals(Result.PASS));
+ assertFalse(Result.STOP.equals(Result.FAIL));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index dcf8fb4..5196641 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -964,6 +964,50 @@
</test>
<test> <!-- TESTED -->
+ <description>help: help for find</description>
+ <test-commands>
+ <command>-help find</command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>-find <path> \.\.\. <expression> \.\.\. :
+ Finds all files that match the specified expression and
+ applies selected actions to them\. If no <path> is specified
+ then defaults to the current working directory\. If no
+ expression is specified then defaults to -print\.
+
+ The following primary expressions are recognised:
+ -name pattern
+ -iname pattern
+ Evaluates as true if the basename of the file matches the
+ pattern using standard file system globbing\.
+ If -iname is used then the match is case insensitive\.
+
+ -print
+ -print0
+ Always evaluates to true. Causes the current pathname to be
+ written to standard output followed by a newline. If the -print0
+ expression is used then an ASCII NULL character is appended rather
+ than a newline.
+
+ The following operators are recognised:
+ expression -a expression
+ expression -and expression
+ expression expression
+ Logical AND operator for joining two expressions\. Returns
+ true if both child expressions return true\. Implied by the
+ juxtaposition of two expressions and so does not need to be
+ explicitly specified\. The second expression will not be
+ applied if the first fails\.
+</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
<description>help: help for help</description>
<test-commands>
<command>-help help</command>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba879a5d/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 8939f87..c86b06d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -16841,5 +16841,228 @@
</comparator>
</comparators>
</test>
+
+ <!-- Tests for find -->
+ <test> <!-- TESTED -->
+ <description>find: default expression</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /donotfind</command>
+ <command>-fs NAMENODE -mkdir donotfind</command>
+ <command>-fs NAMENODE -mkdir /findtest</command>
+ <command>-fs NAMENODE -mkdir /findtest/item1</command>
+ <command>-fs NAMENODE -mkdir /findtest/item1/item1a</command>
+ <command>-fs NAMENODE -touchz /findtest/item1/item1a/item1aa</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /findtest/item1/item1b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /findtest/item2</command>
+ <command>-fs NAMENODE -mkdir /findtest/item3</command>
+ <command>-fs NAMENODE -mkdir /findtest/item4</command>
+ <command>-fs NAMENODE -mkdir /findtest/item4/item4a</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data120bytes /findtest/item4/item4b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data1k /findtest/item5</command>
+ <command>-fs NAMENODE -find /findtest</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /donotfind</command>
+ <command>-fs NAMENODE -rm -r donotfind</command>
+ <command>-fs NAMENODE -rm -r /findtest</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>^/findtest
+/findtest/item1
+/findtest/item1/item1a
+/findtest/item1/item1a/item1aa
+/findtest/item1/item1b
+/findtest/item2
+/findtest/item3
+/findtest/item4
+/findtest/item4/item4a
+/findtest/item4/item4b
+/findtest/item5
+$</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test> <!-- TESTED -->
+ <description>find: -print </description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /donotfind</command>
+ <command>-fs NAMENODE -mkdir donotfind</command>
+ <command>-fs NAMENODE -mkdir /findtest</command>
+ <command>-fs NAMENODE -mkdir /findtest/item1</command>
+ <command>-fs NAMENODE -mkdir /findtest/item1/item1a</command>
+ <command>-fs NAMENODE -touchz /findtest/item1/item1a/item1aa</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /findtest/item1/item1b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /findtest/item2</command>
+ <command>-fs NAMENODE -mkdir /findtest/item3</command>
+ <command>-fs NAMENODE -mkdir /findtest/item4</command>
+ <command>-fs NAMENODE -mkdir /findtest/item4/item4a</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data120bytes /findtest/item4/item4b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data1k /findtest/item5</command>
+ <command>-fs NAMENODE -find /findtest -print</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /donotfind</command>
+ <command>-fs NAMENODE -rm -r donotfind</command>
+ <command>-fs NAMENODE -rm -r /findtest</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>^/findtest
+/findtest/item1
+/findtest/item1/item1a
+/findtest/item1/item1a/item1aa
+/findtest/item1/item1b
+/findtest/item2
+/findtest/item3
+/findtest/item4
+/findtest/item4/item4a
+/findtest/item4/item4b
+/findtest/item5
+$</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test> <!-- TESTED -->
+ <description>find: -print (relative path) </description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /donotfind</command>
+ <command>-fs NAMENODE -mkdir -p donotfind</command>
+ <command>-fs NAMENODE -mkdir -p findtest</command>
+ <command>-fs NAMENODE -mkdir -p findtest/item1</command>
+ <command>-fs NAMENODE -mkdir -p findtest/item1/item1a</command>
+ <command>-fs NAMENODE -touchz findtest/item1/item1a/item1aa</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes findtest/item1/item1b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes findtest/item2</command>
+ <command>-fs NAMENODE -mkdir -p findtest/item3</command>
+ <command>-fs NAMENODE -mkdir -p findtest/item4</command>
+ <command>-fs NAMENODE -mkdir -p findtest/item4/item4a</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data120bytes findtest/item4/item4b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data1k findtest/item5</command>
+ <command>-fs NAMENODE -find findtest -print</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /donotfind</command>
+ <command>-fs NAMENODE -rm -r donotfind</command>
+ <command>-fs NAMENODE -rm -r findtest</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>^findtest
+findtest/item1
+findtest/item1/item1a
+findtest/item1/item1a/item1aa
+findtest/item1/item1b
+findtest/item2
+findtest/item3
+findtest/item4
+findtest/item4/item4a
+findtest/item4/item4b
+findtest/item5
+$</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test> <!-- TESTED -->
+ <description>find: -print (cwd) </description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /donotfind</command>
+ <command>-fs NAMENODE -mkdir findtest</command>
+ <command>-fs NAMENODE -mkdir findtest/item1</command>
+ <command>-fs NAMENODE -mkdir findtest/item1/item1a</command>
+ <command>-fs NAMENODE -touchz findtest/item1/item1a/item1aa</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes findtest/item1/item1b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes findtest/item2</command>
+ <command>-fs NAMENODE -mkdir findtest/item3</command>
+ <command>-fs NAMENODE -mkdir findtest/item4</command>
+ <command>-fs NAMENODE -mkdir findtest/item4/item4a</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data120bytes findtest/item4/item4b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data1k findtest/item5</command>
+ <command>-fs NAMENODE -find -print</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r findtest</command>
+ <command>-fs NAMENODE -rm -r /donotfind</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>^.
+findtest
+findtest/item1
+findtest/item1/item1a
+findtest/item1/item1a/item1aa
+findtest/item1/item1b
+findtest/item2
+findtest/item3
+findtest/item4
+findtest/item4/item4a
+findtest/item4/item4b
+findtest/item5
+$</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test> <!-- TESTED -->
+ <description>find: -name </description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /findtest</command>
+ <command>-fs NAMENODE -mkdir /findtest/item1</command>
+ <command>-fs NAMENODE -mkdir /findtest/item1/item1a</command>
+ <command>-fs NAMENODE -touchz /findtest/item1/item1a/item1aa</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /findtest/item1/item1b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /findtest/item2</command>
+ <command>-fs NAMENODE -mkdir /findtest/item3</command>
+ <command>-fs NAMENODE -mkdir /findtest/item4</command>
+ <command>-fs NAMENODE -mkdir /findtest/item4/item4a</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data120bytes /findtest/item4/item4b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data1k /findtest/item5</command>
+ <command>-fs NAMENODE -find /findtest -name item*a</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /findtest</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>^/findtest/item1/item1a
+/findtest/item1/item1a/item1aa
+/findtest/item4/item4a
+$</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test> <!-- TESTED -->
+ <description>find: -iname </description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /findtest</command>
+ <command>-fs NAMENODE -mkdir /findtest/item1</command>
+ <command>-fs NAMENODE -mkdir /findtest/item1/item1a</command>
+ <command>-fs NAMENODE -touchz /findtest/item1/item1a/item1aa</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /findtest/item1/item1b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /findtest/item2</command>
+ <command>-fs NAMENODE -mkdir /findtest/item3</command>
+ <command>-fs NAMENODE -mkdir /findtest/item4</command>
+ <command>-fs NAMENODE -mkdir /findtest/item4/item4a</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data120bytes /findtest/item4/item4b</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data1k /findtest/item5</command>
+ <command>-fs NAMENODE -find /findtest -iname ITEM*a</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /findtest</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>^/findtest/item1/item1a
+/findtest/item1/item1a/item1aa
+/findtest/item4/item4a
+$</expected-output>
+ </comparator>
+ </comparators>
+ </test>
</tests>
</configuration>
[19/45] hadoop git commit: YARN-2853. Fixed a bug in ResourceManager
causing apps to hang when the user kill request races with ApplicationMaster
finish. Contributed by Jian He.
Posted by vi...@apache.org.
YARN-2853. Fixed a bug in ResourceManager causing apps to hang when the user kill request races with ApplicationMaster finish. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3651fe1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3651fe1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3651fe1b
Branch: refs/heads/HDFS-EC
Commit: 3651fe1b089851b38be351c00a9899817166bf3e
Parents: 33ea5ae
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu Nov 13 08:12:41 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Nov 13 08:13:03 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../ApplicationMasterService.java | 1 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 24 +++-
.../yarn/server/resourcemanager/TestRM.java | 111 +++++++++++++++++++
.../rmapp/TestRMAppTransitions.java | 6 -
5 files changed, 136 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3651fe1b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 326d33c..32e9e19 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -87,6 +87,9 @@ Release 2.7.0 - UNRELEASED
YARN-2603. ApplicationConstants missing HADOOP_MAPRED_HOME (Ray Chiang via
aw)
+ YARN-2853. Fixed a bug in ResourceManager causing apps to hang when the user
+ kill request races with ApplicationMaster finish. (Jian He via vinodkv)
+
Release 2.6.0 - 2014-11-15
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3651fe1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 64988f5..d0b199f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -348,6 +348,7 @@ public class ApplicationMasterService extends AbstractService implements
// ApplicationDoesNotExistInCacheException before and after
// RM work-preserving restart.
if (rmApp.isAppFinalStateStored()) {
+ LOG.info(rmApp.getApplicationId() + " unregistered successfully. ");
return FinishApplicationMasterResponse.newInstance(true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3651fe1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index ad92cc4..aeb2cda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -293,13 +293,23 @@ public class RMAppImpl implements RMApp, Recoverable {
RMAppEventType.ATTEMPT_KILLED,
new FinalSavingTransition(
new AppKilledTransition(), RMAppState.KILLED))
+ .addTransition(RMAppState.KILLING, RMAppState.FINAL_SAVING,
+ RMAppEventType.ATTEMPT_UNREGISTERED,
+ new FinalSavingTransition(
+ new AttemptUnregisteredTransition(),
+ RMAppState.FINISHING, RMAppState.FINISHED))
+ .addTransition(RMAppState.KILLING, RMAppState.FINISHED,
+ // UnManagedAM directly jumps to finished
+ RMAppEventType.ATTEMPT_FINISHED, FINISHED_TRANSITION)
+ .addTransition(RMAppState.KILLING,
+ EnumSet.of(RMAppState.FINAL_SAVING),
+ RMAppEventType.ATTEMPT_FAILED,
+ new AttemptFailedTransition(RMAppState.KILLING))
+
.addTransition(RMAppState.KILLING, RMAppState.KILLING,
EnumSet.of(
RMAppEventType.NODE_UPDATE,
RMAppEventType.ATTEMPT_REGISTERED,
- RMAppEventType.ATTEMPT_UNREGISTERED,
- RMAppEventType.ATTEMPT_FINISHED,
- RMAppEventType.ATTEMPT_FAILED,
RMAppEventType.APP_UPDATE_SAVED,
RMAppEventType.KILL, RMAppEventType.MOVE))
@@ -1199,6 +1209,14 @@ public class RMAppImpl implements RMApp, Recoverable {
+ app.maxAppAttempts);
if (!app.submissionContext.getUnmanagedAM()
&& numberOfFailure < app.maxAppAttempts) {
+ if (initialState.equals(RMAppState.KILLING)) {
+ // If this is not last attempt, app should be killed instead of
+ // launching a new attempt
+ app.rememberTargetTransitionsAndStoreState(event,
+ new AppKilledTransition(), RMAppState.KILLED, RMAppState.KILLED);
+ return RMAppState.FINAL_SAVING;
+ }
+
boolean transferStateFromPreviousAttempt;
RMAppFailedAttemptEvent failedEvent = (RMAppFailedAttemptEvent) event;
transferStateFromPreviousAttempt =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3651fe1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
index 4865420..77d8cdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import org.junit.Before;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.spy;
import java.util.ArrayList;
@@ -37,16 +38,19 @@ import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -57,6 +61,8 @@ import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityMockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -73,6 +79,8 @@ import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.junit.Test;
import org.mockito.ArgumentMatcher;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
@SuppressWarnings({"unchecked", "rawtypes"})
public class TestRM extends ParameterizedSchedulerTestBase {
@@ -638,4 +646,107 @@ public class TestRM extends ParameterizedSchedulerTestBase {
Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
}
+ // Test Kill an app while the app is finishing in the meanwhile.
+ @Test (timeout = 30000)
+ public void testKillFinishingApp() throws Exception{
+
+ // this dispatcher ignores RMAppAttemptEventType.KILL event
+ final Dispatcher dispatcher = new AsyncDispatcher() {
+ @Override
+ public EventHandler getEventHandler() {
+
+ class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
+ @Override
+ public boolean matches(Object argument) {
+ if (argument instanceof RMAppAttemptEvent) {
+ if (((RMAppAttemptEvent) argument).getType().equals(
+ RMAppAttemptEventType.KILL)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ EventHandler handler = spy(super.getEventHandler());
+ doNothing().when(handler).handle(argThat(new EventArgMatcher()));
+ return handler;
+ }
+ };
+
+ MockRM rm1 = new MockRM(conf){
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
+ };
+ rm1.start();
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
+ nm1.registerNode();
+ RMApp app1 = rm1.submitApp(200);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ rm1.killApp(app1.getApplicationId());
+
+ FinishApplicationMasterRequest req =
+ FinishApplicationMasterRequest.newInstance(
+ FinalApplicationStatus.SUCCEEDED, "", "");
+ am1.unregisterAppAttempt(req,true);
+
+ rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHING);
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
+ rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
+ rm1.waitForState(app1.getApplicationId(), RMAppState.FINISHED);
+ }
+
+ // Test Kill an app while the app is failing
+ @Test (timeout = 30000)
+ public void testKillFailingApp() throws Exception{
+
+ // this dispatcher ignores RMAppAttemptEventType.KILL event
+ final Dispatcher dispatcher = new AsyncDispatcher() {
+ @Override
+ public EventHandler getEventHandler() {
+
+ class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
+ @Override
+ public boolean matches(Object argument) {
+ if (argument instanceof RMAppAttemptEvent) {
+ if (((RMAppAttemptEvent) argument).getType().equals(
+ RMAppAttemptEventType.KILL)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ EventHandler handler = spy(super.getEventHandler());
+ doNothing().when(handler).handle(argThat(new EventArgMatcher()));
+ return handler;
+ }
+ };
+
+ MockRM rm1 = new MockRM(conf){
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
+ };
+ rm1.start();
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
+ nm1.registerNode();
+ RMApp app1 = rm1.submitApp(200);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ rm1.killApp(app1.getApplicationId());
+
+ // fail the app by sending container_finished event.
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
+ rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
+ // app is killed, not launching a new attempt
+ rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3651fe1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index bbfb0ee..e68d074 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -726,12 +726,6 @@ public class TestRMAppTransitions {
application.handle(event);
rmDispatcher.await();
- // Ignore Attempt_Finished if we were supposed to go to Finished.
- assertAppState(RMAppState.KILLING, application);
- RMAppEvent finishEvent =
- new RMAppFinishedAttemptEvent(application.getApplicationId(), null);
- application.handle(finishEvent);
- assertAppState(RMAppState.KILLING, application);
sendAttemptUpdateSavedEvent(application);
sendAppUpdateSavedEvent(application);
assertKilled(application);
[18/45] hadoop git commit: YARN-2846. Incorrect persist exit code for
running containers in reacquireContainer() that interrupted by NodeManager
restart. Contributed by Junping Du
Posted by vi...@apache.org.
YARN-2846. Incorrect persist exit code for running containers in reacquireContainer() that interrupted by NodeManager restart. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33ea5ae9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33ea5ae9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33ea5ae9
Branch: refs/heads/HDFS-EC
Commit: 33ea5ae92b9dd3abace104903d9a94d17dd75af5
Parents: 177e809
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Nov 13 16:11:04 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Nov 13 16:11:04 2014 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 ++++
.../server/nodemanager/ContainerExecutor.java | 21 +++++++-------------
.../nodemanager/LinuxContainerExecutor.java | 2 +-
.../launcher/RecoveredContainerLaunch.java | 20 ++++++++++++-------
4 files changed, 25 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33ea5ae9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 01dc246..326d33c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -955,6 +955,10 @@ Release 2.6.0 - 2014-11-15
YARN-2794. Fixed log messages about distributing system-credentials. (Jian He via
zjshen)
+ YARN-2846. Incorrect persist exit code for running containers in
+ reacquireContainer() that interrupted by NodeManager restart. (Junping Du
+ via jlowe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33ea5ae9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 8133413..327f882 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -159,9 +159,10 @@ public abstract class ContainerExecutor implements Configurable {
* @param containerId The ID of the container to reacquire
* @return The exit code of the pre-existing container
* @throws IOException
+ * @throws InterruptedException
*/
public int reacquireContainer(String user, ContainerId containerId)
- throws IOException {
+ throws IOException, InterruptedException {
Path pidPath = getPidFilePath(containerId);
if (pidPath == null) {
LOG.warn(containerId + " is not active, returning terminated error");
@@ -175,13 +176,8 @@ public abstract class ContainerExecutor implements Configurable {
}
LOG.info("Reacquiring " + containerId + " with pid " + pid);
- try {
- while(isContainerProcessAlive(user, pid)) {
- Thread.sleep(1000);
- }
- } catch (InterruptedException e) {
- throw new IOException("Interrupted while waiting for process " + pid
- + " to exit", e);
+ while(isContainerProcessAlive(user, pid)) {
+ Thread.sleep(1000);
}
// wait for exit code file to appear
@@ -194,12 +190,9 @@ public abstract class ContainerExecutor implements Configurable {
LOG.info(containerId + " was deactivated");
return ExitCode.TERMINATED.getExitCode();
}
- try {
- Thread.sleep(sleepMsec);
- } catch (InterruptedException e) {
- throw new IOException(
- "Interrupted while waiting for exit code from " + containerId, e);
- }
+
+ Thread.sleep(sleepMsec);
+
msecLeft -= sleepMsec;
}
if (msecLeft < 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33ea5ae9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 60cb058..4db4ef2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -347,7 +347,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
@Override
public int reacquireContainer(String user, ContainerId containerId)
- throws IOException {
+ throws IOException, InterruptedException {
try {
return super.reacquireContainer(user, containerId);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33ea5ae9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
index 446695a..03a39aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
@@ -73,6 +73,7 @@ public class RecoveredContainerLaunch extends ContainerLaunch {
dispatcher.getEventHandler().handle(new ContainerEvent(containerId,
ContainerEventType.CONTAINER_LAUNCHED));
+ boolean notInterrupted = true;
try {
File pidFile = locatePidFile(appIdStr, containerIdStr);
if (pidFile != null) {
@@ -85,14 +86,19 @@ public class RecoveredContainerLaunch extends ContainerLaunch {
}
} catch (IOException e) {
LOG.error("Unable to recover container " + containerIdStr, e);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for exit code from " + containerId);
+ notInterrupted = false;
} finally {
- this.completed.set(true);
- exec.deactivateContainer(containerId);
- try {
- getContext().getNMStateStore().storeContainerCompleted(containerId,
- retCode);
- } catch (IOException e) {
- LOG.error("Unable to set exit code for container " + containerId);
+ if (notInterrupted) {
+ this.completed.set(true);
+ exec.deactivateContainer(containerId);
+ try {
+ getContext().getNMStateStore().storeContainerCompleted(containerId,
+ retCode);
+ } catch (IOException e) {
+ LOG.error("Unable to set exit code for container " + containerId);
+ }
}
}
[11/45] hadoop git commit: YARN-2603. ApplicationConstants missing
HADOOP_MAPRED_HOME (Ray Chiang via aw)
Posted by vi...@apache.org.
YARN-2603. ApplicationConstants missing HADOOP_MAPRED_HOME (Ray Chiang via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0a9cd37
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0a9cd37
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0a9cd37
Branch: refs/heads/HDFS-EC
Commit: b0a9cd37ab9ef45ed9a71b04f1634fa54dd4c438
Parents: d7150a1
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Nov 12 17:02:53 2014 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Nov 12 17:02:53 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/api/ApplicationConstants.java | 5 +++++
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a9cd37/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1f991f6..01dc246 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -84,6 +84,9 @@ Release 2.7.0 - UNRELEASED
YARN-2713. "RM Home" link in NM should point to one of the RMs in an
HA setup. (kasha)
+ YARN-2603. ApplicationConstants missing HADOOP_MAPRED_HOME (Ray Chiang via
+ aw)
+
Release 2.6.0 - 2014-11-15
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a9cd37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index d42cd67..93438b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -181,6 +181,11 @@ public interface ApplicationConstants {
HADOOP_HDFS_HOME("HADOOP_HDFS_HOME"),
/**
+ * $HADOOP_MAPRED_HOME
+ */
+ HADOOP_MAPRED_HOME("HADOOP_MAPRED_HOME"),
+
+ /**
* $MALLOC_ARENA_MAX
*/
MALLOC_ARENA_MAX("MALLOC_ARENA_MAX"),
[08/45] hadoop git commit: HDFS-7387. Merging to branch-2.6 for
hadoop-2.6.0-rc1.
Posted by vi...@apache.org.
HDFS-7387. Merging to branch-2.6 for hadoop-2.6.0-rc1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/782abbb0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/782abbb0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/782abbb0
Branch: refs/heads/HDFS-EC
Commit: 782abbb000ab1c9e2e033e347eea8827d6e866ef
Parents: be7bf95
Author: Arun C. Murthy <ac...@apache.org>
Authored: Wed Nov 12 11:32:44 2014 -0800
Committer: Arun C. Murthy <ac...@apache.org>
Committed: Wed Nov 12 11:32:44 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/782abbb0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ea89344..cd8814e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -413,9 +413,6 @@ Release 2.7.0 - UNRELEASED
HDFS-7366. BlockInfo should take replication as an short in the constructor.
(Li Lu via wheat9)
- HDFS-7387. NFS may only do partial commit due to a race between COMMIT and write
- (brandonli)
-
HDFS-7389. Named user ACL cannot stop the user from accessing the FS entity.
(Vinayakumar B via cnauroth)
@@ -1430,6 +1427,9 @@ Release 2.6.0 - 2014-11-15
HDFS-7383. DataNode.requestShortCircuitFdsForRead may throw
NullPointerException. (szetszwo via suresh)
+ HDFS-7387. NFS may only do partial commit due to a race between COMMIT and write
+ (brandonli)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
[34/45] hadoop git commit: YARN-2857. ConcurrentModificationException
in ContainerLogAppender. Contributed by Mohammad Kamrul Islam
Posted by vi...@apache.org.
YARN-2857. ConcurrentModificationException in ContainerLogAppender. Contributed by Mohammad Kamrul Islam
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2fe8a80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2fe8a80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2fe8a80
Branch: refs/heads/HDFS-EC
Commit: f2fe8a800e5b0f3875931adba9ae20c6a95aa4ff
Parents: 1a1dcce
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Nov 14 20:47:26 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Nov 14 20:47:26 2014 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/ContainerLogAppender.java | 7 ++-
.../hadoop/yarn/TestContainerLogAppender.java | 46 ++++++++++++++++++++
3 files changed, 55 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2fe8a80/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4373813..34568ff 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -87,6 +87,9 @@ Release 2.7.0 - UNRELEASED
YARN-2856. Fixed RMAppImpl to handle ATTEMPT_KILLED event at ACCEPTED state
on app recovery. (Rohith Sharmaks via jianhe)
+ YARN-2857. ConcurrentModificationException in ContainerLogAppender
+ (Mohammad Kamrul Islam via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2fe8a80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
index 95876ef..0aba8b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
@@ -41,6 +41,7 @@ public class ContainerLogAppender extends FileAppender
//so that log4j can configure it from the configuration(log4j.properties).
private int maxEvents;
private Queue<LoggingEvent> tail = null;
+ private boolean closing = false;
@Override
public void activateOptions() {
@@ -57,6 +58,9 @@ public class ContainerLogAppender extends FileAppender
@Override
public void append(LoggingEvent event) {
synchronized (this) {
+ if (closing) { // When closing drop any new/transitive CLA appending
+ return;
+ }
if (tail == null) {
super.append(event);
} else {
@@ -77,8 +81,9 @@ public class ContainerLogAppender extends FileAppender
@Override
public synchronized void close() {
+ closing = true;
if (tail != null) {
- for(LoggingEvent event: tail) {
+ for (LoggingEvent event : tail) {
super.append(event);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2fe8a80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
new file mode 100644
index 0000000..3b20d18
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn;
+
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.junit.Test;
+
+public class TestContainerLogAppender {
+
+ @Test
+ public void testAppendInClose() throws Exception {
+ final ContainerLogAppender claAppender = new ContainerLogAppender();
+ claAppender.setName("testCLA");
+ claAppender.setLayout(new PatternLayout("%-5p [%t]: %m%n"));
+ claAppender.setContainerLogDir("target/testAppendInClose/logDir");
+ claAppender.setTotalLogFileSize(1000);
+ claAppender.activateOptions();
+ final Logger claLog = Logger.getLogger("testAppendInClose-catergory");
+ claLog.setAdditivity(false);
+ claLog.addAppender(claAppender);
+ claLog.info(new Object() {
+ public String toString() {
+ claLog.info("message1");
+ return "return message1";
+ }
+ });
+ claAppender.close();
+ }
+}
[44/45] hadoop git commit: HDFS-7394. Log at INFO level,
not WARN level,
when InvalidToken is seen in ShortCircuitCache (Keith Pak via Colin P. McCabe)
Posted by vi...@apache.org.
HDFS-7394. Log at INFO level, not WARN level, when InvalidToken is seen in ShortCircuitCache (Keith Pak via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26d3b7e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26d3b7e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26d3b7e6
Branch: refs/heads/HDFS-EC
Commit: 26d3b7e6ae867596bbced0bfddde22d3c0d976bb
Parents: 198fb58
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Sun Nov 16 21:20:53 2014 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Sun Nov 16 21:20:53 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26d3b7e6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2917ea9..53c9c0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -359,6 +359,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7386. Replace check "port number < 1024" with shared isPrivilegedPort
method. (Yongjun Zhang via cnauroth)
+ HDFS-7394. Log at INFO level, not WARN level, when InvalidToken is seen in
+ ShortCircuitCache (Keith Pak via Colin P. McCabe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26d3b7e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
index a4b852f..3edf9a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
@@ -743,7 +743,7 @@ public class ShortCircuitCache implements Closeable {
throw new RetriableException("interrupted");
}
if (info.getInvalidTokenException() != null) {
- LOG.warn(this + ": could not get " + key + " due to InvalidToken " +
+ LOG.info(this + ": could not get " + key + " due to InvalidToken " +
"exception.", info.getInvalidTokenException());
return info;
}
@@ -802,7 +802,7 @@ public class ShortCircuitCache implements Closeable {
Waitable<ShortCircuitReplicaInfo> waitableInMap = replicaInfoMap.get(key);
if (waitableInMap == newWaitable) replicaInfoMap.remove(key);
if (info.getInvalidTokenException() != null) {
- LOG.warn(this + ": could not load " + key + " due to InvalidToken " +
+ LOG.info(this + ": could not load " + key + " due to InvalidToken " +
"exception.", info.getInvalidTokenException());
} else {
LOG.warn(this + ": failed to load " + key);
[38/45] hadoop git commit: YARN-2816. NM fail to start with NPE
during container recovery. Contributed by Zhihai Xu
Posted by vi...@apache.org.
YARN-2816. NM fail to start with NPE during container recovery. Contributed by Zhihai Xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49c38898
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49c38898
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49c38898
Branch: refs/heads/HDFS-EC
Commit: 49c38898b0be64fc686d039ed2fb2dea1378df02
Parents: 3baaa42
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Nov 14 21:25:59 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Nov 14 21:25:59 2014 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../recovery/NMLeveldbStateStoreService.java | 24 +++++++++++++++++++-
.../TestNMLeveldbStateStoreService.java | 7 ++++++
3 files changed, 33 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49c38898/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 34568ff..1f8af55 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -90,6 +90,9 @@ Release 2.7.0 - UNRELEASED
YARN-2857. ConcurrentModificationException in ContainerLogAppender
(Mohammad Kamrul Islam via jlowe)
+ YARN-2816. NM fail to start with NPE during container recovery (Zhihai Xu
+ via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49c38898/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 7cf4921..9d54688 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -146,6 +146,8 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
throws IOException {
ArrayList<RecoveredContainerState> containers =
new ArrayList<RecoveredContainerState>();
+ ArrayList<ContainerId> containersToRemove =
+ new ArrayList<ContainerId>();
LeveldbIterator iter = null;
try {
iter = new LeveldbIterator(db);
@@ -165,7 +167,14 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
ContainerId containerId = ConverterUtils.toContainerId(
key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos));
String keyPrefix = key.substring(0, idEndPos+1);
- containers.add(loadContainerState(containerId, iter, keyPrefix));
+ RecoveredContainerState rcs = loadContainerState(containerId,
+ iter, keyPrefix);
+ // Don't load container without StartContainerRequest
+ if (rcs.startRequest != null) {
+ containers.add(rcs);
+ } else {
+ containersToRemove.add(containerId);
+ }
}
} catch (DBException e) {
throw new IOException(e);
@@ -175,6 +184,19 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
}
}
+ // remove container without StartContainerRequest
+ for (ContainerId containerId : containersToRemove) {
+ LOG.warn("Remove container " + containerId +
+ " with incomplete records");
+ try {
+ removeContainer(containerId);
+ // TODO: kill and cleanup the leaked container
+ } catch (IOException e) {
+ LOG.error("Unable to remove container " + containerId +
+ " in store", e);
+ }
+ }
+
return containers;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49c38898/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index 438cec3..f7f43cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -274,6 +274,13 @@ public class TestNMLeveldbStateStoreService {
assertEquals(containerReq, rcs.getStartRequest());
assertTrue(rcs.getDiagnostics().isEmpty());
+ // store a new container record without StartContainerRequest
+ ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 6);
+ stateStore.storeContainerLaunched(containerId1);
+ recoveredContainers = stateStore.loadContainersState();
+ // check whether the new container record is discarded
+ assertEquals(1, recoveredContainers.size());
+
// launch the container, add some diagnostics, and verify recovered
StringBuilder diags = new StringBuilder();
stateStore.storeContainerLaunched(containerId);
[06/45] hadoop git commit: YARN-2843. Merging to branch-2.6 for
hadoop-2.6.0-rc1.
Posted by vi...@apache.org.
YARN-2843. Merging to branch-2.6 for hadoop-2.6.0-rc1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51957ca8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51957ca8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51957ca8
Branch: refs/heads/HDFS-EC
Commit: 51957ca81886ac3553c44848631fdf53766a25ae
Parents: 3e7a515
Author: Arun C. Murthy <ac...@apache.org>
Authored: Wed Nov 12 10:41:24 2014 -0800
Committer: Arun C. Murthy <ac...@apache.org>
Committed: Wed Nov 12 10:41:24 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51957ca8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 78fd6ce..f869fde 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -84,9 +84,6 @@ Release 2.7.0 - UNRELEASED
YARN-2713. "RM Home" link in NM should point to one of the RMs in an
HA setup. (kasha)
- YARN-2843. Fixed NodeLabelsManager to trim inputs for hosts and labels so
- as to make them work correctly. (Wangda Tan via vinodkv)
-
Release 2.6.0 - 2014-11-15
INCOMPATIBLE CHANGES
@@ -949,6 +946,9 @@ Release 2.6.0 - 2014-11-15
YARN-2841. RMProxy should retry EOFException. (Jian He via xgong)
+ YARN-2843. Fixed NodeLabelsManager to trim inputs for hosts and labels so
+ as to make them work correctly. (Wangda Tan via vinodkv)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
[04/45] hadoop git commit: HADOOP-9576. Merging to branch-2.6 for
hadoop-2.6.0-rc1.
Posted by vi...@apache.org.
HADOOP-9576. Merging to branch-2.6 for hadoop-2.6.0-rc1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44feec71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44feec71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44feec71
Branch: refs/heads/HDFS-EC
Commit: 44feec7122f48c6ff6ec1a90347098447b67543b
Parents: a041430
Author: Arun C. Murthy <ac...@apache.org>
Authored: Wed Nov 12 09:55:28 2014 -0800
Committer: Arun C. Murthy <ac...@apache.org>
Committed: Wed Nov 12 09:55:28 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44feec71/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 26c39cf..480737b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -422,9 +422,6 @@ Release 2.7.0 - UNRELEASED
HADOOP-11294. Nfs3FileAttributes should not change the values of rdev,
nlink and size in the constructor. (Brandon Li via wheat9)
- HADOOP-9576. Changed NetUtils#wrapException to throw EOFException instead
- of wrapping it as IOException. (Steve Loughran via jianhe)
-
Release 2.6.0 - 2014-11-15
INCOMPATIBLE CHANGES
@@ -1103,6 +1100,9 @@ Release 2.6.0 - 2014-11-15
HADOOP-11282. Skip NFS TestShellBasedIdMapping tests that are irrelevant on
Windows. (cnauroth)
+ HADOOP-9576. Changed NetUtils#wrapException to throw EOFException instead
+ of wrapping it as IOException. (Steve Loughran via jianhe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
[21/45] hadoop git commit: YARN-2635. Merging to branch-2.6 for
hadoop-2.6.0-rc1.
Posted by vi...@apache.org.
YARN-2635. Merging to branch-2.6 for hadoop-2.6.0-rc1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81dc0ac6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81dc0ac6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81dc0ac6
Branch: refs/heads/HDFS-EC
Commit: 81dc0ac6dcf2f34ad607da815ea0144f178691a9
Parents: d648e60
Author: Arun C. Murthy <ac...@apache.org>
Authored: Thu Nov 13 10:27:47 2014 -0800
Committer: Arun C. Murthy <ac...@apache.org>
Committed: Thu Nov 13 10:27:47 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81dc0ac6/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2179faf..8f679fd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -53,9 +53,6 @@ Release 2.7.0 - UNRELEASED
YARN-1979. TestDirectoryCollection fails when the umask is unusual.
(Vinod Kumar Vavilapalli and Tsuyoshi OZAWA via junping_du)
- YARN-2635. TestRM, TestRMRestart, TestClientToAMTokens should run
- with both CS and FS. (Wei Yan and kasha via kasha)
-
YARN-2641. Decommission nodes on -refreshNodes instead of next
NM-RM heartbeat. (Zhihai Xu via kasha)
@@ -459,6 +456,9 @@ Release 2.6.0 - 2014-11-15
YARN-2818. Removed the now unnecessary user entity injection from Timeline
service given we now have domains. (Zhijie Shen via vinodkv)
+ YARN-2635. TestRM, TestRMRestart, TestClientToAMTokens should run
+ with both CS and FS. (Wei Yan and kasha via kasha)
+
OPTIMIZATIONS
BUG FIXES
[42/45] hadoop git commit: YARN-2432. RMStateStore should process the
pending events before close. Contributed by Varun Saxena
Posted by vi...@apache.org.
YARN-2432. RMStateStore should process the pending events before close. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b860661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b860661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b860661
Branch: refs/heads/HDFS-EC
Commit: 9b860661a5754e829bd8c34ef092757421589bbe
Parents: 1925e2a
Author: Jian He <ji...@apache.org>
Authored: Fri Nov 14 17:24:09 2014 -0800
Committer: Jian He <ji...@apache.org>
Committed: Fri Nov 14 17:24:09 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b860661/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cfd9459..9d3179b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -93,6 +93,9 @@ Release 2.7.0 - UNRELEASED
YARN-2816. NM fail to start with NPE during container recovery (Zhihai Xu
via jlowe)
+ YARN-2432. RMStateStore should process the pending events before close.
+ (Varun Saxena via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b860661/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index beac403..8948b54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -492,8 +492,8 @@ public abstract class RMStateStore extends AbstractService {
@Override
protected void serviceStop() throws Exception {
- closeInternal();
dispatcher.stop();
+ closeInternal();
}
/**
[16/45] hadoop git commit: HADOOP-11291. Log the cause of SASL
connection failures. Contributed by Stephen Chu.
Posted by vi...@apache.org.
HADOOP-11291. Log the cause of SASL connection failures. Contributed by Stephen Chu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dae5b5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dae5b5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dae5b5a
Branch: refs/heads/HDFS-EC
Commit: 7dae5b5a880205fd0125c143c44498db79c19392
Parents: 0abb973
Author: cnauroth <cn...@apache.org>
Authored: Wed Nov 12 21:54:06 2014 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Wed Nov 12 21:54:06 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dae5b5a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1a304d7..732cdc7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -380,6 +380,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-10786. Fix UGI#reloginFromKeytab on Java 8. (Stephen Chu via wheat9)
+ HADOOP-11291. Log the cause of SASL connection failures.
+ (Stephen Chu via cnauroth)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dae5b5a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 84fe552..45a4660 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -668,7 +668,7 @@ public class Client {
String msg = "Couldn't setup connection for "
+ UserGroupInformation.getLoginUser().getUserName() + " to "
+ remoteId;
- LOG.warn(msg);
+ LOG.warn(msg, ex);
throw (IOException) new IOException(msg).initCause(ex);
}
} else {
[37/45] hadoop git commit: MAPREDUCE-6161. mapred hsadmin command
missing from trunk. Contributed by Allen Wittenauer
Posted by vi...@apache.org.
MAPREDUCE-6161. mapred hsadmin command missing from trunk. Contributed by Allen Wittenauer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3baaa429
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3baaa429
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3baaa429
Branch: refs/heads/HDFS-EC
Commit: 3baaa42945fd432e10c547d3ead57e220f64cd57
Parents: f95017b
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Nov 14 21:10:06 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Nov 14 21:10:06 2014 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
hadoop-mapreduce-project/bin/mapred | 10 +++++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3baaa429/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 28595ca..653ad44 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -160,6 +160,9 @@ Trunk (Unreleased)
MAPREDUCE-5972. Fix typo 'programatically' in job.xml (and a few other
places) (Akira AJISAKA via aw)
+ MAPREDUCE-6161. mapred hsadmin command missing from trunk (Allen Wittenauer
+ via jlowe)
+
BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3baaa429/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index 1fa5d44..9131ba7 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -25,8 +25,9 @@ function hadoop_usage
echo " classpath prints the class path needed for running"
echo " mapreduce subcommands"
echo " distcp <srcurl> <desturl> copy file or directories recursively"
- echo " job manipulate MapReduce jobs"
echo " historyserver run job history servers as a standalone daemon"
+ echo " hsadmin job history server admin interface"
+ echo " job manipulate MapReduce jobs"
echo " pipes run a Pipes job"
echo " queue get information regarding JobQueues"
echo " sampler sampler"
@@ -98,8 +99,15 @@ case ${COMMAND} in
fi
HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_JHS_LOGGER:-$HADOOP_DAEMON_ROOT_LOGGER}
;;
+ hsadmin)
+ CLASS=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
job)
CLASS=org.apache.hadoop.mapred.JobClient
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
pipes)
CLASS=org.apache.hadoop.mapred.pipes.Submitter
[03/45] hadoop git commit: YARN-2236. [YARN-1492] Shared Cache
uploader service on the Node Manager. (Chris Trezzo and Sanjin Lee via kasha)
Posted by vi...@apache.org.
YARN-2236. [YARN-1492] Shared Cache uploader service on the Node Manager. (Chris Trezzo and Sanjin Lee via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0414303
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0414303
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0414303
Branch: refs/heads/HDFS-EC
Commit: a04143039e7fe310d807f40584633096181cfada
Parents: f8aefa5
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Nov 12 09:31:05 2014 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Nov 12 09:31:05 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/api/records/LocalResource.java | 38 +++
.../hadoop/yarn/conf/YarnConfiguration.java | 19 ++
.../src/main/proto/yarn_protos.proto | 1 +
.../records/impl/pb/LocalResourcePBImpl.java | 20 ++
.../yarn/sharedcache/ChecksumSHA256Impl.java | 37 +++
.../yarn/sharedcache/SharedCacheChecksum.java | 43 +++
.../sharedcache/SharedCacheChecksumFactory.java | 84 ++++++
.../org/apache/hadoop/yarn/util/FSDownload.java | 5 +-
.../src/main/resources/yarn-default.xml | 18 ++
.../hadoop/yarn/server/utils/BuilderUtils.java | 10 +-
.../containermanager/ContainerManagerImpl.java | 13 +
.../container/ContainerImpl.java | 72 ++++-
.../localizer/LocalResourceRequest.java | 11 +
.../sharedcache/SharedCacheUploadEvent.java | 58 ++++
.../sharedcache/SharedCacheUploadEventType.java | 28 ++
.../sharedcache/SharedCacheUploadService.java | 126 ++++++++
.../sharedcache/SharedCacheUploader.java | 289 +++++++++++++++++++
.../container/TestContainer.java | 2 +-
.../TestResourceLocalizationService.java | 2 +-
.../TestSharedCacheUploadService.java | 50 ++++
.../sharedcache/TestSharedCacheUploader.java | 241 ++++++++++++++++
22 files changed, 1158 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 05bb8f6..ec3a331 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -45,6 +45,9 @@ Release 2.7.0 - UNRELEASED
YARN-2186. [YARN-1492] Node Manager uploader service for cache manager.
(Chris Trezzo and Sangjin Lee via kasha)
+ YARN-2236. [YARN-1492] Shared Cache uploader service on the Node
+ Manager. (Chris Trezzo and Sanjin Lee via kasha)
+
IMPROVEMENTS
YARN-1979. TestDirectoryCollection fails when the umask is unusual.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
index f14a136..726d969 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.util.Records;
@@ -48,6 +49,14 @@ public abstract class LocalResource {
public static LocalResource newInstance(URL url, LocalResourceType type,
LocalResourceVisibility visibility, long size, long timestamp,
String pattern) {
+ return newInstance(url, type, visibility, size, timestamp, pattern, false);
+ }
+
+ @Public
+ @Unstable
+ public static LocalResource newInstance(URL url, LocalResourceType type,
+ LocalResourceVisibility visibility, long size, long timestamp,
+ String pattern, boolean shouldBeUploadedToSharedCache) {
LocalResource resource = Records.newRecord(LocalResource.class);
resource.setResource(url);
resource.setType(type);
@@ -55,6 +64,7 @@ public abstract class LocalResource {
resource.setSize(size);
resource.setTimestamp(timestamp);
resource.setPattern(pattern);
+ resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
return resource;
}
@@ -65,6 +75,15 @@ public abstract class LocalResource {
return newInstance(url, type, visibility, size, timestamp, null);
}
+ @Public
+ @Unstable
+ public static LocalResource newInstance(URL url, LocalResourceType type,
+ LocalResourceVisibility visibility, long size, long timestamp,
+ boolean shouldBeUploadedToSharedCache) {
+ return newInstance(url, type, visibility, size, timestamp, null,
+ shouldBeUploadedToSharedCache);
+ }
+
/**
* Get the <em>location</em> of the resource to be localized.
* @return <em>location</em> of the resource to be localized
@@ -170,4 +189,23 @@ public abstract class LocalResource {
@Public
@Stable
public abstract void setPattern(String pattern);
+
+ /**
+ * NM uses it to decide whether if it is necessary to upload the resource to
+ * the shared cache
+ */
+ @Public
+ @Unstable
+ public abstract boolean getShouldBeUploadedToSharedCache();
+
+ /**
+ * Inform NM whether upload to SCM is needed.
+ *
+ * @param shouldBeUploadedToSharedCache <em>shouldBeUploadedToSharedCache</em>
+ * of this request
+ */
+ @Public
+ @Unstable
+ public abstract void setShouldBeUploadedToSharedCache(
+ boolean shouldBeUploadedToSharedCache);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9b57a42..4b4f581 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1472,6 +1472,25 @@ public class YarnConfiguration extends Configuration {
SHARED_CACHE_PREFIX + "uploader.server.thread-count";
public static final int DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT = 50;
+ /** the checksum algorithm implementation **/
+ public static final String SHARED_CACHE_CHECKSUM_ALGO_IMPL =
+ SHARED_CACHE_PREFIX + "checksum.algo.impl";
+ public static final String DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL =
+ "org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl";
+
+ // node manager (uploader) configs
+ /**
+ * The replication factor for the node manager uploader for the shared cache.
+ */
+ public static final String SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR =
+ SHARED_CACHE_PREFIX + "nm.uploader.replication.factor";
+ public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR =
+ 10;
+
+ public static final String SHARED_CACHE_NM_UPLOADER_THREAD_COUNT =
+ SHARED_CACHE_PREFIX + "nm.uploader.thread-count";
+ public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = 20;
+
////////////////////////////////
// Other Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5c86c2d..c4e756d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -159,6 +159,7 @@ message LocalResourceProto {
optional LocalResourceTypeProto type = 4;
optional LocalResourceVisibilityProto visibility = 5;
optional string pattern = 6;
+ optional bool should_be_uploaded_to_shared_cache = 7;
}
message ApplicationResourceUsageReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
index 16bd597..560b081 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
@@ -192,6 +192,26 @@ public class LocalResourcePBImpl extends LocalResource {
builder.setPattern(pattern);
}
+ @Override
+ public synchronized boolean getShouldBeUploadedToSharedCache() {
+ LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasShouldBeUploadedToSharedCache()) {
+ return false;
+ }
+ return p.getShouldBeUploadedToSharedCache();
+ }
+
+ @Override
+ public synchronized void setShouldBeUploadedToSharedCache(
+ boolean shouldBeUploadedToSharedCache) {
+ maybeInitBuilder();
+ if (!shouldBeUploadedToSharedCache) {
+ builder.clearShouldBeUploadedToSharedCache();
+ return;
+ }
+ builder.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
+ }
+
private LocalResourceTypeProto convertToProtoFormat(LocalResourceType e) {
return ProtoUtils.convertToProtoFormat(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumSHA256Impl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumSHA256Impl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumSHA256Impl.java
new file mode 100644
index 0000000..24ceeae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/ChecksumSHA256Impl.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sharedcache;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+@Private
+@Evolving
+/**
+ * The SHA-256 implementation of the shared cache checksum interface.
+ */
+public class ChecksumSHA256Impl implements SharedCacheChecksum {
+ public String computeChecksum(InputStream in) throws IOException {
+ return DigestUtils.sha256Hex(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksum.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksum.java
new file mode 100644
index 0000000..7e6fdda
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksum.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sharedcache;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+@Public
+@Evolving
+/**
+ * An interface to calculate a checksum for a resource in the shared cache. The
+ * checksum implementation should be thread safe.
+ */
+public interface SharedCacheChecksum {
+
+ /**
+ * Calculate the checksum of the passed input stream.
+ *
+ * @param in <code>InputStream</code> to be checksumed
+ * @return the message digest of the input stream
+ * @throws IOException
+ */
+ public String computeChecksum(InputStream in) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksumFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksumFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksumFactory.java
new file mode 100644
index 0000000..cbfd95d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/sharedcache/SharedCacheChecksumFactory.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.sharedcache;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+@SuppressWarnings("unchecked")
+@Public
+@Evolving
+/**
+ * A factory class for creating checksum objects based on a configurable
+ * algorithm implementation
+ */
+public class SharedCacheChecksumFactory {
+ private static final
+ ConcurrentMap<Class<? extends SharedCacheChecksum>,SharedCacheChecksum>
+ instances =
+ new ConcurrentHashMap<Class<? extends SharedCacheChecksum>,
+ SharedCacheChecksum>();
+
+ private static final Class<? extends SharedCacheChecksum> defaultAlgorithm;
+
+ static {
+ try {
+ defaultAlgorithm = (Class<? extends SharedCacheChecksum>)
+ Class.forName(
+ YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
+ } catch (Exception e) {
+ // cannot happen
+ throw new ExceptionInInitializerError(e);
+ }
+ }
+
+ /**
+ * Get a new <code>SharedCacheChecksum</code> object based on the configurable
+ * algorithm implementation
+ * (see <code>yarn.sharedcache.checksum.algo.impl</code>)
+ *
+ * @return <code>SharedCacheChecksum</code> object
+ */
+ public static SharedCacheChecksum getChecksum(Configuration conf) {
+ Class<? extends SharedCacheChecksum> clazz =
+ conf.getClass(YarnConfiguration.SHARED_CACHE_CHECKSUM_ALGO_IMPL,
+ defaultAlgorithm, SharedCacheChecksum.class);
+ SharedCacheChecksum checksum = instances.get(clazz);
+ if (checksum == null) {
+ try {
+ checksum = ReflectionUtils.newInstance(clazz, conf);
+ SharedCacheChecksum old = instances.putIfAbsent(clazz, checksum);
+ if (old != null) {
+ checksum = old;
+ }
+ } catch (Exception e) {
+ throw new YarnRuntimeException(e);
+ }
+ }
+
+ return checksum;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index 2737cce..436cb31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -32,6 +32,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
@@ -134,8 +135,8 @@ public class FSDownload implements Callable<Path> {
* @return true if the path in the current path is visible to all, false
* otherwise
*/
- @VisibleForTesting
- static boolean isPublic(FileSystem fs, Path current, FileStatus sStat,
+ @Private
+ public static boolean isPublic(FileSystem fs, Path current, FileStatus sStat,
LoadingCache<Path,Future<FileStatus>> statCache) throws IOException {
current = fs.makeQualified(current);
//the leaf level file should be readable by others
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index af4a5eb..af3b5aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1458,6 +1458,24 @@
<value>50</value>
</property>
+ <property>
+ <description>The algorithm used to compute checksums of files (SHA-256 by default)</description>
+ <name>yarn.sharedcache.checksum.algo.impl</name>
+ <value>org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl</value>
+ </property>
+
+ <property>
+ <description>The replication factor for the node manager uploader for the shared cache (10 by default)</description>
+ <name>yarn.sharedcache.nm.uploader.replication.factor</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <description>The number of threads used to upload files from a node manager instance (20 by default)</description>
+ <name>yarn.sharedcache.nm.uploader.thread-count</name>
+ <value>20</value>
+ </property>
+
<!-- Other configuration -->
<property>
<description>The interval that the yarn client library uses to poll the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index a7e5d9c..1b32671 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -70,6 +70,7 @@ import com.google.common.annotations.VisibleForTesting;
* Builder utilities to construct various objects.
*
*/
+@Private
public class BuilderUtils {
private static final RecordFactory recordFactory = RecordFactoryProvider
@@ -94,7 +95,8 @@ public class BuilderUtils {
}
public static LocalResource newLocalResource(URL url, LocalResourceType type,
- LocalResourceVisibility visibility, long size, long timestamp) {
+ LocalResourceVisibility visibility, long size, long timestamp,
+ boolean shouldBeUploadedToSharedCache) {
LocalResource resource =
recordFactory.newRecordInstance(LocalResource.class);
resource.setResource(url);
@@ -102,14 +104,15 @@ public class BuilderUtils {
resource.setVisibility(visibility);
resource.setSize(size);
resource.setTimestamp(timestamp);
+ resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
return resource;
}
public static LocalResource newLocalResource(URI uri,
LocalResourceType type, LocalResourceVisibility visibility, long size,
- long timestamp) {
+ long timestamp, boolean shouldBeUploadedToSharedCache) {
return newLocalResource(ConverterUtils.getYarnUrlFromURI(uri), type,
- visibility, size, timestamp);
+ visibility, size, timestamp, shouldBeUploadedToSharedCache);
}
public static ApplicationId newApplicationId(RecordFactory recordFactory,
@@ -245,7 +248,6 @@ public class BuilderUtils {
return newToken(Token.class, identifier, kind, password, service);
}
- @Private
@VisibleForTesting
public static Token newContainerToken(NodeId nodeId,
byte[] password, ContainerTokenIdentifier tokenIdentifier) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 35b232f..bb277d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -119,6 +119,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.NonAggregatingLogHandler;
@@ -227,6 +229,13 @@ public class ContainerManagerImpl extends CompositeService implements
addIfService(logHandler);
dispatcher.register(LogHandlerEventType.class, logHandler);
+ // add the shared cache upload service (it will do nothing if the shared
+ // cache is disabled)
+ SharedCacheUploadService sharedCacheUploader =
+ createSharedCacheUploaderService();
+ addService(sharedCacheUploader);
+ dispatcher.register(SharedCacheUploadEventType.class, sharedCacheUploader);
+
waitForContainersOnShutdownMillis =
conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS) +
@@ -367,6 +376,10 @@ public class ContainerManagerImpl extends CompositeService implements
deletionContext, dirsHandler, context);
}
+ protected SharedCacheUploadService createSharedCacheUploaderService() {
+ return new SharedCacheUploadService();
+ }
+
protected ContainersLauncher createContainersLauncher(Context context,
ContainerExecutor exec) {
return new ContainersLauncher(context, this.dispatcher, exec, dirsHandler, this);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index fa54ee1..6b65a54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -27,6 +27,7 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -59,6 +60,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStartMonitoringEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStopMonitoringEvent;
@@ -104,6 +107,10 @@ public class ContainerImpl implements Container {
new ArrayList<LocalResourceRequest>();
private final List<LocalResourceRequest> appRsrcs =
new ArrayList<LocalResourceRequest>();
+ private final Map<LocalResourceRequest, Path> resourcesToBeUploaded =
+ new ConcurrentHashMap<LocalResourceRequest, Path>();
+ private final Map<LocalResourceRequest, Boolean> resourcesUploadPolicies =
+ new ConcurrentHashMap<LocalResourceRequest, Boolean>();
// whether container has been recovered after a restart
private RecoveredContainerStatus recoveredStatus =
@@ -637,6 +644,8 @@ public class ContainerImpl implements Container {
container.pendingResources.put(req, links);
}
links.add(rsrc.getKey());
+ storeSharedCacheUploadPolicy(container, req, rsrc.getValue()
+ .getShouldBeUploadedToSharedCache());
switch (rsrc.getValue().getVisibility()) {
case PUBLIC:
container.publicRsrcs.add(req);
@@ -686,30 +695,76 @@ public class ContainerImpl implements Container {
}
/**
+ * Store the resource's shared cache upload policies
+ * Given LocalResourceRequest can be shared across containers in
+ * LocalResourcesTrackerImpl, we preserve the upload policies here.
+ * In addition, it is possible for the application to create several
+ * "identical" LocalResources as part of
+ * ContainerLaunchContext.setLocalResources with different symlinks.
+ * There is a corner case where these "identical" local resources have
+ * different upload policies. For that scenario, upload policy will be set to
+ * true as long as there is at least one LocalResource entry with
+ * upload policy set to true.
+ */
+ private static void storeSharedCacheUploadPolicy(ContainerImpl container,
+ LocalResourceRequest resourceRequest, Boolean uploadPolicy) {
+ Boolean storedUploadPolicy =
+ container.resourcesUploadPolicies.get(resourceRequest);
+ if (storedUploadPolicy == null || (!storedUploadPolicy && uploadPolicy)) {
+ container.resourcesUploadPolicies.put(resourceRequest, uploadPolicy);
+ }
+ }
+
+ /**
* Transition when one of the requested resources for this container
* has been successfully localized.
*/
static class LocalizedTransition implements
MultipleArcTransition<ContainerImpl,ContainerEvent,ContainerState> {
+ @SuppressWarnings("unchecked")
@Override
public ContainerState transition(ContainerImpl container,
ContainerEvent event) {
ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event;
- List<String> syms =
- container.pendingResources.remove(rsrcEvent.getResource());
+ LocalResourceRequest resourceRequest = rsrcEvent.getResource();
+ Path location = rsrcEvent.getLocation();
+ List<String> syms = container.pendingResources.remove(resourceRequest);
if (null == syms) {
- LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
+ LOG.warn("Localized unknown resource " + resourceRequest +
" for container " + container.containerId);
assert false;
// fail container?
return ContainerState.LOCALIZING;
}
- container.localizedResources.put(rsrcEvent.getLocation(), syms);
+ container.localizedResources.put(location, syms);
+
+ // check to see if this resource should be uploaded to the shared cache
+ // as well
+ if (shouldBeUploadedToSharedCache(container, resourceRequest)) {
+ container.resourcesToBeUploaded.put(resourceRequest, location);
+ }
if (!container.pendingResources.isEmpty()) {
return ContainerState.LOCALIZING;
}
container.sendLaunchEvent();
+
+ // If this is a recovered container that has already launched, skip
+ // uploading resources to the shared cache. We do this to avoid uploading
+ // the same resources multiple times. The tradeoff is that in the case of
+ // a recovered container, there is a chance that resources don't get
+ // uploaded into the shared cache. This is OK because resources are not
+ // acknowledged by the SCM until they have been uploaded by the node
+ // manager.
+ if (container.recoveredStatus != RecoveredContainerStatus.LAUNCHED
+ && container.recoveredStatus != RecoveredContainerStatus.COMPLETED) {
+ // kick off uploads to the shared cache
+ container.dispatcher.getEventHandler().handle(
+ new SharedCacheUploadEvent(container.resourcesToBeUploaded, container
+ .getLaunchContext(), container.getUser(),
+ SharedCacheUploadEventType.UPLOAD));
+ }
+
container.metrics.endInitingContainer();
return ContainerState.LOCALIZED;
}
@@ -1018,4 +1073,13 @@ public class ContainerImpl implements Container {
private boolean hasDefaultExitCode() {
return (this.exitCode == ContainerExitStatus.INVALID);
}
+
+ /**
+ * Returns whether the specific resource should be uploaded to the shared
+ * cache.
+ */
+ private static boolean shouldBeUploadedToSharedCache(ContainerImpl container,
+ LocalResourceRequest resource) {
+ return container.resourcesUploadPolicies.get(resource);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
index 70bead7..607d0b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
@@ -152,6 +152,17 @@ public class LocalResourceRequest
}
@Override
+ public boolean getShouldBeUploadedToSharedCache() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setShouldBeUploadedToSharedCache(
+ boolean shouldBeUploadedToSharedCache) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void setResource(URL resource) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEvent.java
new file mode 100644
index 0000000..2be080e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEvent.java
@@ -0,0 +1,58 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache;
+
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
+
+@Private
+@Unstable
+public class SharedCacheUploadEvent extends
+ AbstractEvent<SharedCacheUploadEventType> {
+ private final Map<LocalResourceRequest,Path> resources;
+ private final ContainerLaunchContext context;
+ private final String user;
+
+ public SharedCacheUploadEvent(Map<LocalResourceRequest,Path> resources,
+ ContainerLaunchContext context, String user,
+ SharedCacheUploadEventType eventType) {
+ super(eventType);
+ this.resources = resources;
+ this.context = context;
+ this.user = user;
+ }
+
+ public Map<LocalResourceRequest,Path> getResources() {
+ return resources;
+ }
+
+ public ContainerLaunchContext getContainerLaunchContext() {
+ return context;
+ }
+
+ public String getUser() {
+ return user;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEventType.java
new file mode 100644
index 0000000..5ba7e1b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadEventType.java
@@ -0,0 +1,28 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
+public enum SharedCacheUploadEventType {
+ UPLOAD
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java
new file mode 100644
index 0000000..cb11f99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java
@@ -0,0 +1,126 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.api.SCMUploaderProtocol;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+@Private
+@Unstable
+/**
+ * Service that uploads localized files to the shared cache. The upload is
+ * considered not critical, and is done on a best-effort basis. Failure to
+ * upload is not fatal.
+ */
+public class SharedCacheUploadService extends AbstractService implements
+ EventHandler<SharedCacheUploadEvent> {
+ private static final Log LOG =
+ LogFactory.getLog(SharedCacheUploadService.class);
+
+ private boolean enabled;
+ private FileSystem fs;
+ private FileSystem localFs;
+ private ExecutorService uploaderPool;
+ private SCMUploaderProtocol scmClient;
+
+ public SharedCacheUploadService() {
+ super(SharedCacheUploadService.class.getName());
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ enabled = conf.getBoolean(YarnConfiguration.SHARED_CACHE_ENABLED,
+ YarnConfiguration.DEFAULT_SHARED_CACHE_ENABLED);
+ if (enabled) {
+ int threadCount =
+ conf.getInt(YarnConfiguration.SHARED_CACHE_NM_UPLOADER_THREAD_COUNT,
+ YarnConfiguration.DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT);
+ uploaderPool = Executors.newFixedThreadPool(threadCount,
+ new ThreadFactoryBuilder().
+ setNameFormat("Shared cache uploader #%d").
+ build());
+ scmClient = createSCMClient(conf);
+ try {
+ fs = FileSystem.get(conf);
+ localFs = FileSystem.getLocal(conf);
+ } catch (IOException e) {
+ LOG.error("Unexpected exception in getting the filesystem", e);
+ throw new RuntimeException(e);
+ }
+ }
+ super.serviceInit(conf);
+ }
+
+ private SCMUploaderProtocol createSCMClient(Configuration conf) {
+ YarnRPC rpc = YarnRPC.create(conf);
+ InetSocketAddress scmAddress =
+ conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
+ YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS,
+ YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);
+ return (SCMUploaderProtocol)rpc.getProxy(
+ SCMUploaderProtocol.class, scmAddress, conf);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (enabled) {
+ uploaderPool.shutdown();
+ RPC.stopProxy(scmClient);
+ }
+ super.serviceStop();
+ }
+
+ @Override
+ public void handle(SharedCacheUploadEvent event) {
+ if (enabled) {
+ Map<LocalResourceRequest,Path> resources = event.getResources();
+ for (Map.Entry<LocalResourceRequest,Path> e: resources.entrySet()) {
+ SharedCacheUploader uploader =
+ new SharedCacheUploader(e.getKey(), e.getValue(), event.getUser(),
+ getConfig(), scmClient, fs, localFs);
+ // fire off an upload task
+ uploaderPool.submit(uploader);
+ }
+ }
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
new file mode 100644
index 0000000..050d531
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
@@ -0,0 +1,289 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.net.URISyntaxException;
+import java.util.Random;
+import java.util.concurrent.Callable;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.api.SCMUploaderProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.SCMUploaderNotifyRequest;
+import org.apache.hadoop.yarn.server.sharedcache.SharedCacheUtil;
+import org.apache.hadoop.yarn.sharedcache.SharedCacheChecksum;
+import org.apache.hadoop.yarn.sharedcache.SharedCacheChecksumFactory;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.FSDownload;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * The callable class that handles the actual upload to the shared cache.
+ */
+class SharedCacheUploader implements Callable<Boolean> {
+ // rwxr-xr-x
+ static final FsPermission DIRECTORY_PERMISSION =
+ new FsPermission((short)00755);
+ // r-xr-xr-x
+ static final FsPermission FILE_PERMISSION =
+ new FsPermission((short)00555);
+
+ private static final Log LOG = LogFactory.getLog(SharedCacheUploader.class);
+ private static final ThreadLocal<Random> randomTl =
+ new ThreadLocal<Random>() {
+ @Override
+ protected Random initialValue() {
+ return new Random(System.nanoTime());
+ }
+ };
+
+ private final LocalResource resource;
+ private final Path localPath;
+ private final String user;
+ private final Configuration conf;
+ private final SCMUploaderProtocol scmClient;
+ private final FileSystem fs;
+ private final FileSystem localFs;
+ private final String sharedCacheRootDir;
+ private final int nestedLevel;
+ private final SharedCacheChecksum checksum;
+ private final RecordFactory recordFactory;
+
+ public SharedCacheUploader(LocalResource resource, Path localPath,
+ String user, Configuration conf, SCMUploaderProtocol scmClient)
+ throws IOException {
+ this(resource, localPath, user, conf, scmClient,
+ FileSystem.get(conf), localPath.getFileSystem(conf));
+ }
+
+ /**
+ * @param resource the local resource that contains the original remote path
+ * @param localPath the path in the local filesystem where the resource is
+ * localized
+ * @param fs the filesystem of the shared cache
+ * @param localFs the local filesystem
+ */
+ public SharedCacheUploader(LocalResource resource, Path localPath,
+ String user, Configuration conf, SCMUploaderProtocol scmClient,
+ FileSystem fs, FileSystem localFs) {
+ this.resource = resource;
+ this.localPath = localPath;
+ this.user = user;
+ this.conf = conf;
+ this.scmClient = scmClient;
+ this.fs = fs;
+ this.sharedCacheRootDir =
+ conf.get(YarnConfiguration.SHARED_CACHE_ROOT,
+ YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT);
+ this.nestedLevel = SharedCacheUtil.getCacheDepth(conf);
+ this.checksum = SharedCacheChecksumFactory.getChecksum(conf);
+ this.localFs = localFs;
+ this.recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ }
+
+ /**
+ * Uploads the file under the shared cache, and notifies the shared cache
+ * manager. If it is unable to upload the file because it already exists, it
+ * returns false.
+ */
+ @Override
+ public Boolean call() throws Exception {
+ Path tempPath = null;
+ try {
+ if (!verifyAccess()) {
+ LOG.warn("User " + user + " is not authorized to upload file " +
+ localPath.getName());
+ return false;
+ }
+
+ // first determine the actual local path that will be used for upload
+ Path actualPath = getActualPath();
+ // compute the checksum
+ String checksumVal = computeChecksum(actualPath);
+ // create the directory (if it doesn't exist)
+ Path directoryPath =
+ new Path(SharedCacheUtil.getCacheEntryPath(nestedLevel,
+ sharedCacheRootDir, checksumVal));
+ // let's not check if the directory already exists: in the vast majority
+ // of the cases, the directory does not exist; as long as mkdirs does not
+ // error out if it exists, we should be fine
+ fs.mkdirs(directoryPath, DIRECTORY_PERMISSION);
+ // create the temporary file
+ tempPath = new Path(directoryPath, getTemporaryFileName(actualPath));
+ if (!uploadFile(actualPath, tempPath)) {
+ LOG.warn("Could not copy the file to the shared cache at " + tempPath);
+ return false;
+ }
+
+ // set the permission so that it is readable but not writable
+ fs.setPermission(tempPath, FILE_PERMISSION);
+ // rename it to the final filename
+ Path finalPath = new Path(directoryPath, actualPath.getName());
+ if (!fs.rename(tempPath, finalPath)) {
+ LOG.warn("The file already exists under " + finalPath +
+ ". Ignoring this attempt.");
+ deleteTempFile(tempPath);
+ return false;
+ }
+
+ // notify the SCM
+ if (!notifySharedCacheManager(checksumVal, actualPath.getName())) {
+ // the shared cache manager rejected the upload (as it is likely
+ // uploaded under a different name
+ // clean up this file and exit
+ fs.delete(finalPath, false);
+ return false;
+ }
+
+ // set the replication factor
+ short replication =
+ (short)conf.getInt(YarnConfiguration.SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR,
+ YarnConfiguration.DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR);
+ fs.setReplication(finalPath, replication);
+ LOG.info("File " + actualPath.getName() +
+ " was uploaded to the shared cache at " + finalPath);
+ return true;
+ } catch (IOException e) {
+ LOG.warn("Exception while uploading the file " + localPath.getName(), e);
+ // in case an exception is thrown, delete the temp file
+ deleteTempFile(tempPath);
+ throw e;
+ }
+ }
+
+ @VisibleForTesting
+ Path getActualPath() throws IOException {
+ Path path = localPath;
+ FileStatus status = localFs.getFileStatus(path);
+ if (status != null && status.isDirectory()) {
+ // for certain types of resources that get unpacked, the original file may
+ // be found under the directory with the same name (see
+ // FSDownload.unpack); check if the path is a directory and if so look
+ // under it
+ path = new Path(path, path.getName());
+ }
+ return path;
+ }
+
+ private void deleteTempFile(Path tempPath) {
+ try {
+ if (tempPath != null && fs.exists(tempPath)) {
+ fs.delete(tempPath, false);
+ }
+ } catch (IOException ignore) {}
+ }
+
+ /**
+ * Checks that the (original) remote file is either owned by the user who
+ * started the app or public.
+ */
+ @VisibleForTesting
+ boolean verifyAccess() throws IOException {
+ // if it is in the public cache, it's trivially OK
+ if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
+ return true;
+ }
+
+ final Path remotePath;
+ try {
+ remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
+ } catch (URISyntaxException e) {
+ throw new IOException("Invalid resource", e);
+ }
+
+ // get the file status of the HDFS file
+ FileSystem remoteFs = remotePath.getFileSystem(conf);
+ FileStatus status = remoteFs.getFileStatus(remotePath);
+ // check to see if the file has been modified in any way
+ if (status.getModificationTime() != resource.getTimestamp()) {
+ LOG.warn("The remote file " + remotePath +
+ " has changed since it's localized; will not consider it for upload");
+ return false;
+ }
+
+ // check for the user ownership
+ if (status.getOwner().equals(user)) {
+ return true; // the user owns the file
+ }
+ // check if the file is publicly readable otherwise
+ return fileIsPublic(remotePath, remoteFs, status);
+ }
+
+ @VisibleForTesting
+ boolean fileIsPublic(final Path remotePath, FileSystem remoteFs,
+ FileStatus status) throws IOException {
+ return FSDownload.isPublic(remoteFs, remotePath, status, null);
+ }
+
+ /**
+ * Uploads the file to the shared cache under a temporary name, and returns
+ * the result.
+ */
+ @VisibleForTesting
+ boolean uploadFile(Path sourcePath, Path tempPath) throws IOException {
+ return FileUtil.copy(localFs, sourcePath, fs, tempPath, false, conf);
+ }
+
+ @VisibleForTesting
+ String computeChecksum(Path path) throws IOException {
+ InputStream is = localFs.open(path);
+ try {
+ return checksum.computeChecksum(is);
+ } finally {
+ try { is.close(); } catch (IOException ignore) {}
+ }
+ }
+
+ private String getTemporaryFileName(Path path) {
+ return path.getName() + "-" + randomTl.get().nextLong();
+ }
+
+ @VisibleForTesting
+ boolean notifySharedCacheManager(String checksumVal, String fileName)
+ throws IOException {
+ try {
+ SCMUploaderNotifyRequest request =
+ recordFactory.newRecordInstance(SCMUploaderNotifyRequest.class);
+ request.setResourceKey(checksumVal);
+ request.setFilename(fileName);
+ return scmClient.notify(request).getAccepted();
+ } catch (YarnException e) {
+ throw new IOException(e);
+ } catch (UndeclaredThrowableException e) {
+ // retrieve the cause of the exception and throw it as an IOException
+ throw new IOException(e.getCause() == null ? e : e.getCause());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 8f7fa78..c28d691 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -642,7 +642,7 @@ public class TestContainer {
URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
LocalResource rsrc =
BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
- r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L);
+ r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
return new SimpleEntry<String, LocalResource>(name, rsrc);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index bf36651..1051e7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -1760,7 +1760,7 @@ public class TestResourceLocalizationService {
URL url = getPath("/local/PRIVATE/" + name);
LocalResource rsrc =
BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
- r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L);
+ r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
return rsrc;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploadService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploadService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploadService.java
new file mode 100644
index 0000000..1b2b2f0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploadService.java
@@ -0,0 +1,50 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache;
+
+import static org.junit.Assert.assertSame;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+
+public class TestSharedCacheUploadService {
+
+ @Test
+ public void testInitDisabled() {
+ testInit(false);
+ }
+
+ @Test
+ public void testInitEnabled() {
+ testInit(true);
+ }
+
+ public void testInit(boolean enabled) {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, enabled);
+
+ SharedCacheUploadService service = new SharedCacheUploadService();
+ service.init(conf);
+ assertSame(enabled, service.isEnabled());
+
+ service.stop();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0414303/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploader.java
new file mode 100644
index 0000000..9234c62
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/TestSharedCacheUploader.java
@@ -0,0 +1,241 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.api.SCMUploaderProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.SCMUploaderNotifyRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.SCMUploaderNotifyResponse;
+import org.junit.Test;
+
+public class TestSharedCacheUploader {
+
+ /**
+ * If verifyAccess fails, the upload should fail
+ */
+ @Test
+ public void testFailVerifyAccess() throws Exception {
+ SharedCacheUploader spied = createSpiedUploader();
+ doReturn(false).when(spied).verifyAccess();
+
+ assertFalse(spied.call());
+ }
+
+ /**
+ * If rename fails, the upload should fail
+ */
+ @Test
+ public void testRenameFail() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
+ LocalResource resource = mock(LocalResource.class);
+ Path localPath = mock(Path.class);
+ when(localPath.getName()).thenReturn("foo.jar");
+ String user = "joe";
+ SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
+ SCMUploaderNotifyResponse response = mock(SCMUploaderNotifyResponse.class);
+ when(response.getAccepted()).thenReturn(true);
+ when(scmClient.notify(isA(SCMUploaderNotifyRequest.class))).
+ thenReturn(response);
+ FileSystem fs = mock(FileSystem.class);
+ // return false when rename is called
+ when(fs.rename(isA(Path.class), isA(Path.class))).thenReturn(false);
+ FileSystem localFs = FileSystem.getLocal(conf);
+ SharedCacheUploader spied =
+ createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
+ localFs);
+ // stub verifyAccess() to return true
+ doReturn(true).when(spied).verifyAccess();
+ // stub getActualPath()
+ doReturn(localPath).when(spied).getActualPath();
+ // stub computeChecksum()
+ doReturn("abcdef0123456789").when(spied).computeChecksum(isA(Path.class));
+ // stub uploadFile() to return true
+ doReturn(true).when(spied).uploadFile(isA(Path.class), isA(Path.class));
+
+ assertFalse(spied.call());
+ }
+
+ /**
+ * If verifyAccess, uploadFile, rename, and notification succeed, the upload
+ * should succeed
+ */
+ @Test
+ public void testSuccess() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
+ LocalResource resource = mock(LocalResource.class);
+ Path localPath = mock(Path.class);
+ when(localPath.getName()).thenReturn("foo.jar");
+ String user = "joe";
+ SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
+ SCMUploaderNotifyResponse response = mock(SCMUploaderNotifyResponse.class);
+ when(response.getAccepted()).thenReturn(true);
+ when(scmClient.notify(isA(SCMUploaderNotifyRequest.class))).
+ thenReturn(response);
+ FileSystem fs = mock(FileSystem.class);
+ // return false when rename is called
+ when(fs.rename(isA(Path.class), isA(Path.class))).thenReturn(true);
+ FileSystem localFs = FileSystem.getLocal(conf);
+ SharedCacheUploader spied =
+ createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
+ localFs);
+ // stub verifyAccess() to return true
+ doReturn(true).when(spied).verifyAccess();
+ // stub getActualPath()
+ doReturn(localPath).when(spied).getActualPath();
+ // stub computeChecksum()
+ doReturn("abcdef0123456789").when(spied).computeChecksum(isA(Path.class));
+ // stub uploadFile() to return true
+ doReturn(true).when(spied).uploadFile(isA(Path.class), isA(Path.class));
+ // stub notifySharedCacheManager to return true
+ doReturn(true).when(spied).notifySharedCacheManager(isA(String.class),
+ isA(String.class));
+
+ assertTrue(spied.call());
+ }
+
+ /**
+ * If verifyAccess, uploadFile, and rename succed, but it receives a nay from
+ * SCM, the file should be deleted
+ */
+ @Test
+ public void testNotifySCMFail() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
+ LocalResource resource = mock(LocalResource.class);
+ Path localPath = mock(Path.class);
+ when(localPath.getName()).thenReturn("foo.jar");
+ String user = "joe";
+ FileSystem fs = mock(FileSystem.class);
+ // return false when rename is called
+ when(fs.rename(isA(Path.class), isA(Path.class))).thenReturn(true);
+ FileSystem localFs = FileSystem.getLocal(conf);
+ SharedCacheUploader spied =
+ createSpiedUploader(resource, localPath, user, conf, null, fs,
+ localFs);
+ // stub verifyAccess() to return true
+ doReturn(true).when(spied).verifyAccess();
+ // stub getActualPath()
+ doReturn(localPath).when(spied).getActualPath();
+ // stub computeChecksum()
+ doReturn("abcdef0123456789").when(spied).computeChecksum(isA(Path.class));
+ // stub uploadFile() to return true
+ doReturn(true).when(spied).uploadFile(isA(Path.class), isA(Path.class));
+ // stub notifySharedCacheManager to return true
+ doReturn(false).when(spied).notifySharedCacheManager(isA(String.class),
+ isA(String.class));
+
+ assertFalse(spied.call());
+ verify(fs).delete(isA(Path.class), anyBoolean());
+ }
+
+ /**
+ * If resource is public, verifyAccess should succeed
+ */
+ @Test
+ public void testVerifyAccessPublicResource() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
+ LocalResource resource = mock(LocalResource.class);
+ // give public visibility
+ when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
+ Path localPath = mock(Path.class);
+ when(localPath.getName()).thenReturn("foo.jar");
+ String user = "joe";
+ SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
+ FileSystem fs = mock(FileSystem.class);
+ FileSystem localFs = FileSystem.getLocal(conf);
+ SharedCacheUploader spied =
+ createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
+ localFs);
+
+ assertTrue(spied.verifyAccess());
+ }
+
+ /**
+ * If the localPath does not exists, getActualPath should get to one level
+ * down
+ */
+ @Test
+ public void testGetActualPath() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
+ LocalResource resource = mock(LocalResource.class);
+ // give public visibility
+ when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
+ Path localPath = new Path("foo.jar");
+ String user = "joe";
+ SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
+ FileSystem fs = mock(FileSystem.class);
+ FileSystem localFs = mock(FileSystem.class);
+ // stub it to return a status that indicates a directory
+ FileStatus status = mock(FileStatus.class);
+ when(status.isDirectory()).thenReturn(true);
+ when(localFs.getFileStatus(localPath)).thenReturn(status);
+ SharedCacheUploader spied =
+ createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
+ localFs);
+
+ Path actualPath = spied.getActualPath();
+ assertEquals(actualPath.getName(), localPath.getName());
+ assertEquals(actualPath.getParent().getName(), localPath.getName());
+ }
+
+ private SharedCacheUploader createSpiedUploader() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
+ LocalResource resource = mock(LocalResource.class);
+ Path localPath = mock(Path.class);
+ String user = "foo";
+ SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
+ FileSystem fs = FileSystem.get(conf);
+ FileSystem localFs = FileSystem.getLocal(conf);
+ return createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
+ localFs);
+ }
+
+ private SharedCacheUploader createSpiedUploader(LocalResource resource, Path localPath,
+ String user, Configuration conf, SCMUploaderProtocol scmClient,
+ FileSystem fs, FileSystem localFs)
+ throws IOException {
+ SharedCacheUploader uploader = new SharedCacheUploader(resource, localPath, user, conf, scmClient,
+ fs, localFs);
+ return spy(uploader);
+ }
+}
[26/45] hadoop git commit: HDFS-7358. Clients may get stuck waiting
when using ByteArrayManager.
Posted by vi...@apache.org.
HDFS-7358. Clients may get stuck waiting when using ByteArrayManager.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/394ba94c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/394ba94c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/394ba94c
Branch: refs/heads/HDFS-EC
Commit: 394ba94c5d2801fbc5d95c7872eeeede28eed1eb
Parents: aee68b6
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu Nov 13 12:28:44 2014 -0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Thu Nov 13 12:28:44 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 61 +++++++++++++++-----
.../hadoop/hdfs/util/ByteArrayManager.java | 15 +++--
.../java/org/apache/hadoop/hdfs/TestHFlush.java | 10 ++--
.../hadoop/hdfs/util/TestByteArrayManager.java | 43 +++++++++-----
5 files changed, 89 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/394ba94c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 858a71a..bd3dcd5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -421,6 +421,9 @@ Release 2.7.0 - UNRELEASED
HDFS-6938. Cleanup javac warnings in FSNamesystem (Charles Lamb via wheat9)
+ HDFS-7358. Clients may get stuck waiting when using ByteArrayManager.
+ (szetszwo)
+
Release 2.6.0 - 2014-11-15
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/394ba94c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 17942f2..51b1006 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -262,7 +262,9 @@ public class DFSOutputStream extends FSOutputSummer
maxChunks = chunksPerPkt;
}
- void writeData(byte[] inarray, int off, int len) {
+ synchronized void writeData(byte[] inarray, int off, int len)
+ throws ClosedChannelException {
+ checkBuffer();
if (dataPos + len > buf.length) {
throw new BufferOverflowException();
}
@@ -270,7 +272,9 @@ public class DFSOutputStream extends FSOutputSummer
dataPos += len;
}
- void writeChecksum(byte[] inarray, int off, int len) {
+ synchronized void writeChecksum(byte[] inarray, int off, int len)
+ throws ClosedChannelException {
+ checkBuffer();
if (len == 0) {
return;
}
@@ -284,7 +288,9 @@ public class DFSOutputStream extends FSOutputSummer
/**
* Write the full packet, including the header, to the given output stream.
*/
- void writeTo(DataOutputStream stm) throws IOException {
+ synchronized void writeTo(DataOutputStream stm) throws IOException {
+ checkBuffer();
+
final int dataLen = dataPos - dataStart;
final int checksumLen = checksumPos - checksumStart;
final int pktLen = HdfsConstants.BYTES_IN_INTEGER + dataLen + checksumLen;
@@ -326,7 +332,13 @@ public class DFSOutputStream extends FSOutputSummer
}
}
- private void releaseBuffer(ByteArrayManager bam) {
+ private synchronized void checkBuffer() throws ClosedChannelException {
+ if (buf == null) {
+ throw new ClosedChannelException();
+ }
+ }
+
+ private synchronized void releaseBuffer(ByteArrayManager bam) {
bam.release(buf);
buf = null;
}
@@ -712,7 +724,7 @@ public class DFSOutputStream extends FSOutputSummer
closeResponder(); // close and join
closeStream();
streamerClosed = true;
- closed = true;
+ setClosed();
synchronized (dataQueue) {
dataQueue.notifyAll();
}
@@ -1616,8 +1628,9 @@ public class DFSOutputStream extends FSOutputSummer
return sock;
}
+ @Override
protected void checkClosed() throws IOException {
- if (closed) {
+ if (isClosed()) {
IOException e = lastException.get();
throw e != null ? e : new ClosedChannelException();
}
@@ -1827,7 +1840,7 @@ public class DFSOutputStream extends FSOutputSummer
synchronized (dataQueue) {
try {
// If queue is full, then wait till we have enough space
- while (!closed && dataQueue.size() + ackQueue.size() > dfsClient.getConf().writeMaxPackets) {
+ while (!isClosed() && dataQueue.size() + ackQueue.size() > dfsClient.getConf().writeMaxPackets) {
try {
dataQueue.wait();
} catch (InterruptedException e) {
@@ -2013,8 +2026,9 @@ public class DFSOutputStream extends FSOutputSummer
// So send an empty sync packet.
currentPacket = createPacket(packetSize, chunksPerPacket,
bytesCurBlock, currentSeqno++);
- } else {
+ } else if (currentPacket != null) {
// just discard the current packet since it is already been sent.
+ currentPacket.releaseBuffer(byteArrayManager);
currentPacket = null;
}
}
@@ -2071,7 +2085,7 @@ public class DFSOutputStream extends FSOutputSummer
} catch (IOException e) {
DFSClient.LOG.warn("Error while syncing", e);
synchronized (this) {
- if (!closed) {
+ if (!isClosed()) {
lastException.set(new IOException("IOException flush:" + e));
closeThreads(true);
}
@@ -2133,7 +2147,7 @@ public class DFSOutputStream extends FSOutputSummer
long begin = Time.monotonicNow();
try {
synchronized (dataQueue) {
- while (!closed) {
+ while (!isClosed()) {
checkClosed();
if (lastAckedSeqno >= seqno) {
break;
@@ -2166,7 +2180,7 @@ public class DFSOutputStream extends FSOutputSummer
* resources associated with this stream.
*/
synchronized void abort() throws IOException {
- if (closed) {
+ if (isClosed()) {
return;
}
streamer.setLastException(new IOException("Lease timeout of "
@@ -2175,6 +2189,25 @@ public class DFSOutputStream extends FSOutputSummer
dfsClient.endFileLease(fileId);
}
+ boolean isClosed() {
+ return closed;
+ }
+
+ void setClosed() {
+ closed = true;
+ synchronized (dataQueue) {
+ releaseBuffer(dataQueue, byteArrayManager);
+ releaseBuffer(ackQueue, byteArrayManager);
+ }
+ }
+
+ private static void releaseBuffer(List<Packet> packets, ByteArrayManager bam) {
+ for(Packet p : packets) {
+ p.releaseBuffer(bam);
+ }
+ packets.clear();
+ }
+
// shutdown datastreamer and responseprocessor threads.
// interrupt datastreamer if force is true
private void closeThreads(boolean force) throws IOException {
@@ -2189,7 +2222,7 @@ public class DFSOutputStream extends FSOutputSummer
} finally {
streamer = null;
s = null;
- closed = true;
+ setClosed();
}
}
@@ -2199,7 +2232,7 @@ public class DFSOutputStream extends FSOutputSummer
*/
@Override
public synchronized void close() throws IOException {
- if (closed) {
+ if (isClosed()) {
IOException e = lastException.getAndSet(null);
if (e == null)
return;
@@ -2229,7 +2262,7 @@ public class DFSOutputStream extends FSOutputSummer
dfsClient.endFileLease(fileId);
} catch (ClosedChannelException e) {
} finally {
- closed = true;
+ setClosed();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/394ba94c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
index 4751e72..ea5e39d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/ByteArrayManager.java
@@ -200,12 +200,7 @@ public abstract class ByteArrayManager {
debugMessage.get().append(", ").append(this);
}
- if (numAllocated == maxAllocated) {
- if (LOG.isDebugEnabled()) {
- debugMessage.get().append(", notifyAll");
- }
- notifyAll();
- }
+ notify();
numAllocated--;
if (numAllocated < 0) {
// it is possible to drop below 0 since
@@ -346,12 +341,13 @@ public abstract class ByteArrayManager {
* the number of allocated arrays drops to below the capacity.
*
* The byte array allocated by this method must be returned for recycling
- * via the {@link ByteArrayManager#recycle(byte[])} method.
+ * via the {@link Impl#release(byte[])} method.
*
* @return a byte array with length larger than or equal to the given length.
*/
@Override
public byte[] newByteArray(final int arrayLength) throws InterruptedException {
+ Preconditions.checkArgument(arrayLength >= 0);
if (LOG.isDebugEnabled()) {
debugMessage.get().append("allocate(").append(arrayLength).append(")");
}
@@ -375,6 +371,7 @@ public abstract class ByteArrayManager {
}
if (LOG.isDebugEnabled()) {
+ debugMessage.get().append(", return byte[").append(array.length).append("]");
logDebugMessage();
}
return array;
@@ -384,7 +381,9 @@ public abstract class ByteArrayManager {
* Recycle the given byte array.
*
* The byte array may or may not be allocated
- * by the {@link ByteArrayManager#allocate(int)} method.
+ * by the {@link Impl#newByteArray(int)} method.
+ *
+ * This is a non-blocking call.
*/
@Override
public int release(final byte[] array) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/394ba94c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHFlush.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHFlush.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHFlush.java
index 7b4d2bb..9ada95f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHFlush.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHFlush.java
@@ -340,11 +340,11 @@ public class TestHFlush {
// If we made it past the hflush(), then that means that the ack made it back
// from the pipeline before we got to the wait() call. In that case we should
// still have interrupted status.
- assertTrue(Thread.currentThread().interrupted());
+ assertTrue(Thread.interrupted());
} catch (InterruptedIOException ie) {
System.out.println("Got expected exception during flush");
}
- assertFalse(Thread.currentThread().interrupted());
+ assertFalse(Thread.interrupted());
// Try again to flush should succeed since we no longer have interrupt status
stm.hflush();
@@ -362,11 +362,11 @@ public class TestHFlush {
// If we made it past the close(), then that means that the ack made it back
// from the pipeline before we got to the wait() call. In that case we should
// still have interrupted status.
- assertTrue(Thread.currentThread().interrupted());
+ assertTrue(Thread.interrupted());
} catch (InterruptedIOException ioe) {
System.out.println("Got expected exception during close");
// If we got the exception, we shouldn't have interrupted status anymore.
- assertFalse(Thread.currentThread().interrupted());
+ assertFalse(Thread.interrupted());
// Now do a successful close.
stm.close();
@@ -374,7 +374,7 @@ public class TestHFlush {
// verify that entire file is good
- AppendTestUtil.checkFullFile(fs, p, fileLen,
+ AppendTestUtil.checkFullFile(fs, p, 4,
fileContents, "Failed to deal with thread interruptions");
} finally {
cluster.shutdown();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/394ba94c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java
index 289617a..77a68c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java
@@ -27,6 +27,8 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -141,7 +143,7 @@ public class TestByteArrayManager {
{ // recycle half of the arrays
for(int i = 0; i < countThreshold/2; i++) {
- recycler.submit(removeLast(allocator.futures));
+ recycler.submit(removeLast(allocator.futures).get());
}
for(Future<Integer> f : recycler.furtures) {
@@ -186,8 +188,8 @@ public class TestByteArrayManager {
}
// recycle an array
- recycler.submit(removeLast(allocator.futures));
- Assert.assertEquals(1, removeLast(recycler.furtures).intValue());
+ recycler.submit(removeLast(allocator.futures).get());
+ Assert.assertEquals(1, removeLast(recycler.furtures).get().intValue());
// check if the thread is unblocked
Thread.sleep(100);
@@ -207,11 +209,11 @@ public class TestByteArrayManager {
}
}
- static <T> T removeLast(List<Future<T>> furtures) throws Exception {
+ static <T> Future<T> removeLast(List<Future<T>> furtures) throws Exception {
return remove(furtures, furtures.size() - 1);
}
- static <T> T remove(List<Future<T>> furtures, int i) throws Exception {
- return furtures.isEmpty()? null: furtures.remove(i).get();
+ static <T> Future<T> remove(List<Future<T>> furtures, int i) throws Exception {
+ return furtures.isEmpty()? null: furtures.remove(i);
}
static <T> void waitForAll(List<Future<T>> furtures) throws Exception {
@@ -320,12 +322,13 @@ public class TestByteArrayManager {
final Runner[] runners = new Runner[Runner.NUM_RUNNERS];
final Thread[] threads = new Thread[runners.length];
- final int num = 1 << 8;
+ final int num = 1 << 10;
for(int i = 0; i < runners.length; i++) {
runners[i] = new Runner(i, countThreshold, countLimit, pool, i, bam);
threads[i] = runners[i].start(num);
}
+ final List<Exception> exceptions = new ArrayList<Exception>();
final Thread randomRecycler = new Thread() {
@Override
public void run() {
@@ -336,10 +339,11 @@ public class TestByteArrayManager {
runners[j].recycle();
} catch (Exception e) {
e.printStackTrace();
- Assert.fail(this + " has " + e);
+ exceptions.add(new Exception(this + " has an exception", e));
}
if ((i & 0xFF) == 0) {
+ LOG.info("randomRecycler sleep, i=" + i);
sleepMs(100);
}
}
@@ -361,6 +365,7 @@ public class TestByteArrayManager {
randomRecycler.start();
randomRecycler.join();
+ Assert.assertTrue(exceptions.isEmpty());
Assert.assertNull(counters.get(0, false));
for(int i = 1; i < runners.length; i++) {
@@ -392,7 +397,7 @@ public class TestByteArrayManager {
}
static class Runner implements Runnable {
- static final int NUM_RUNNERS = 4;
+ static final int NUM_RUNNERS = 5;
static int index2arrayLength(int index) {
return ByteArrayManager.MIN_ARRAY_LENGTH << (index - 1);
@@ -453,16 +458,22 @@ public class TestByteArrayManager {
return f;
}
- byte[] removeFirst() throws Exception {
+ Future<byte[]> removeFirst() throws Exception {
synchronized (arrays) {
return remove(arrays, 0);
}
}
void recycle() throws Exception {
- final byte[] a = removeFirst();
- if (a != null) {
- recycle(a);
+ final Future<byte[]> f = removeFirst();
+ if (f != null) {
+ printf("randomRecycler: ");
+ try {
+ recycle(f.get(10, TimeUnit.MILLISECONDS));
+ } catch(TimeoutException e) {
+ recycle(new byte[maxArrayLength]);
+ printf("timeout, new byte[%d]\n", maxArrayLength);
+ }
}
}
@@ -490,9 +501,9 @@ public class TestByteArrayManager {
submitAllocate();
} else {
try {
- final byte[] a = removeFirst();
- if (a != null) {
- submitRecycle(a);
+ final Future<byte[]> f = removeFirst();
+ if (f != null) {
+ submitRecycle(f.get());
}
} catch (Exception e) {
e.printStackTrace();
[05/45] hadoop git commit: YARN-2841. Merging to branch-2.6 for
hadoop-2.6.0-rc1.
Posted by vi...@apache.org.
YARN-2841. Merging to branch-2.6 for hadoop-2.6.0-rc1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e7a5157
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e7a5157
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e7a5157
Branch: refs/heads/HDFS-EC
Commit: 3e7a5157f05ebc83a57b799518dd2006f204d7a1
Parents: 44feec7
Author: Arun C. Murthy <ac...@apache.org>
Authored: Wed Nov 12 10:36:54 2014 -0800
Committer: Arun C. Murthy <ac...@apache.org>
Committed: Wed Nov 12 10:36:54 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e7a5157/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ec3a331..78fd6ce 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -84,8 +84,6 @@ Release 2.7.0 - UNRELEASED
YARN-2713. "RM Home" link in NM should point to one of the RMs in an
HA setup. (kasha)
- YARN-2841. RMProxy should retry EOFException. (Jian He via xgong)
-
YARN-2843. Fixed NodeLabelsManager to trim inputs for hosts and labels so
as to make them work correctly. (Wangda Tan via vinodkv)
@@ -949,6 +947,8 @@ Release 2.6.0 - 2014-11-15
consistent with the (somewhat incorrect) behaviour in the non-recovery case.
(Jian He via vinodkv)
+ YARN-2841. RMProxy should retry EOFException. (Jian He via xgong)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
[15/45] hadoop git commit: HADOOP-7984. Add hadoop --loglevel option
to change log level. Contributed by Aikira AJISAKA.
Posted by vi...@apache.org.
HADOOP-7984. Add hadoop --loglevel option to change log level. Contributed by Aikira AJISAKA.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0abb973f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0abb973f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0abb973f
Branch: refs/heads/HDFS-EC
Commit: 0abb973f0998ab70e5d3597fcc7ea29ca4d6b583
Parents: e073b61
Author: cnauroth <cn...@apache.org>
Authored: Wed Nov 12 21:41:19 2014 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Wed Nov 12 21:41:19 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/main/bin/hadoop | 2 +-
.../hadoop-common/src/main/bin/hadoop-config.cmd | 16 +++++++++++++++-
.../hadoop-common/src/main/bin/hadoop-config.sh | 6 ++++++
.../hadoop-common/src/main/bin/hadoop-functions.sh | 5 +++--
.../hadoop-common/src/main/bin/hadoop.cmd | 10 +++++++++-
.../src/site/apt/CommandsManual.apt.vm | 7 ++++++-
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 3 ++-
.../hadoop-hdfs/src/main/bin/hdfs.cmd | 10 +++++++++-
.../hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm | 8 ++++++--
hadoop-mapreduce-project/bin/mapred | 3 ++-
hadoop-mapreduce-project/bin/mapred-config.sh | 2 +-
hadoop-mapreduce-project/bin/mapred.cmd | 11 ++++++++++-
hadoop-mapreduce-project/conf/mapred-env.cmd | 2 +-
.../src/site/apt/MapredCommands.apt.vm | 5 ++++-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 ++-
hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh | 2 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 10 +++++++++-
hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd | 2 +-
.../src/site/apt/YarnCommands.apt.vm | 5 ++++-
20 files changed, 95 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f8fca16..1a304d7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -357,6 +357,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-10987. Provide an iterator-based listing API for FileSystem (kihwal)
+ HADOOP-7984. Add hadoop --loglevel option to change log level.
+ (Akira AJISAKA via cnauroth)
+
IMPROVEMENTS
HADOOP-11156. DelegateToFileSystem should implement
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 2a065c2..eccd322 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -17,7 +17,7 @@
function hadoop_usage()
{
- echo "Usage: hadoop [--config confdir] COMMAND"
+ echo "Usage: hadoop [--config confdir] [--loglevel loglevel] COMMAND"
echo " where COMMAND is one of:"
echo " archive -archiveName NAME -p <parent path> <src>* <dest>"
echo " create a Hadoop archive"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd
index d8da5b1..8ae70fb 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd
@@ -88,6 +88,16 @@ if "%1" == "--hosts" (
shift
)
+@rem
+@rem Set log level. Default to INFO.
+@rem
+
+if "%1" == "--loglevel" (
+ set HADOOP_LOGLEVEL=%2
+ shift
+ shift
+)
+
if exist %HADOOP_CONF_DIR%\hadoop-env.cmd (
call %HADOOP_CONF_DIR%\hadoop-env.cmd
)
@@ -157,8 +167,12 @@ if not defined HADOOP_LOGFILE (
set HADOOP_LOGFILE=hadoop.log
)
+if not defined HADOOP_LOGLEVEL (
+ set HADOOP_LOGLEVEL=INFO
+)
+
if not defined HADOOP_ROOT_LOGGER (
- set HADOOP_ROOT_LOGGER=INFO,console
+ set HADOOP_ROOT_LOGGER=%HADOOP_LOGLEVEL%,console
)
@rem
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
index 81a5867..89b0c93 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
@@ -148,6 +148,12 @@ while [[ -z "${_hadoop_common_done}" ]]; do
hadoop_populate_slaves_file "$1"
shift
;;
+ --loglevel)
+ shift
+ # shellcheck disable=SC2034
+ HADOOP_LOGLEVEL="$1"
+ shift
+ ;;
*)
_hadoop_common_done=true
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 52d5af9..4f75a7c 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -155,11 +155,12 @@ function hadoop_basic_init
HADOOP_IDENT_STRING=${HADOP_IDENT_STRING:-$USER}
HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_PREFIX}/logs"}
HADOOP_LOGFILE=${HADOOP_LOGFILE:-hadoop.log}
+ HADOOP_LOGLEVEL=${HADOOP_LOGLEVEL:-INFO}
HADOOP_NICENESS=${HADOOP_NICENESS:-0}
HADOOP_STOP_TIMEOUT=${HADOOP_STOP_TIMEOUT:-5}
HADOOP_PID_DIR=${HADOOP_PID_DIR:-/tmp}
- HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-INFO,console}
- HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_DAEMON_ROOT_LOGGER:-INFO,RFA}
+ HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}
+ HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_DAEMON_ROOT_LOGGER:-${HADOOP_LOGLEVEL},RFA}
HADOOP_SECURITY_LOGGER=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}
HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-1024}
HADOOP_SSH_OPTS=${HADOOP_SSH_OPTS:-"-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
index 2e3e86f..fc2e9b7 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
@@ -88,6 +88,10 @@ call :updatepath %HADOOP_BIN_PATH%
shift
shift
)
+ if "%1" == "--loglevel" (
+ shift
+ shift
+ )
set hadoop-command=%1
if not defined hadoop-command (
@@ -238,6 +242,10 @@ call :updatepath %HADOOP_BIN_PATH%
shift
shift
)
+ if "%1" == "--loglevel" (
+ shift
+ shift
+ )
if [%2] == [] goto :eof
shift
set _arguments=
@@ -256,7 +264,7 @@ call :updatepath %HADOOP_BIN_PATH%
goto :eof
:print_usage
- @echo Usage: hadoop [--config confdir] COMMAND
+ @echo Usage: hadoop [--config confdir] [--loglevel loglevel] COMMAND
@echo where COMMAND is one of:
@echo fs run a generic filesystem user client
@echo version print the version
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
index 1608691..7771aa1 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
@@ -27,7 +27,8 @@ Overview
hadoop script without any arguments prints the description for all
commands.
- Usage: <<<hadoop [--config confdir] [COMMAND] [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
+ Usage: <<<hadoop [--config confdir] [--loglevel loglevel] [COMMAND]
+ [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
Hadoop has an option parsing framework that employs parsing generic
options as well as running classes.
@@ -37,6 +38,10 @@ Overview
*-----------------------+---------------+
| <<<--config confdir>>>| Overwrites the default Configuration directory. Default is <<<${HADOOP_HOME}/conf>>>.
*-----------------------+---------------+
+| <<<--loglevel loglevel>>>| Overwrites the log level. Valid log levels are
+| | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE.
+| | Default is INFO.
+*-----------------------+---------------+
| GENERIC_OPTIONS | The common set of options supported by multiple commands.
| COMMAND_OPTIONS | Various commands with their options are described in the following sections. The commands have been grouped into User Commands and Administration Commands.
*-----------------------+---------------+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index d47df11..36f2e7f 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -17,7 +17,8 @@
function hadoop_usage
{
- echo "Usage: hdfs [--config confdir] [--daemon (start|stop|status)] COMMAND"
+ echo "Usage: hdfs [--config confdir] [--daemon (start|stop|status)]"
+ echo " [--loglevel loglevel] COMMAND"
echo " where COMMAND is one of:"
echo " balancer run a cluster balancing utility"
echo " cacheadmin configure the HDFS cache"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
index 69424ed..00ecfe2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
@@ -34,6 +34,10 @@ if "%1" == "--config" (
shift
shift
)
+if "%1" == "--loglevel" (
+ shift
+ shift
+)
:main
if exist %HADOOP_CONF_DIR%\hadoop-env.cmd (
@@ -165,6 +169,10 @@ goto :eof
shift
shift
)
+ if "%1" == "--loglevel" (
+ shift
+ shift
+ )
if [%2] == [] goto :eof
shift
set _hdfsarguments=
@@ -183,7 +191,7 @@ goto :eof
goto :eof
:print_usage
- @echo Usage: hdfs [--config confdir] COMMAND
+ @echo Usage: hdfs [--config confdir] [--loglevel loglevel] COMMAND
@echo where COMMAND is one of:
@echo dfs run a filesystem command on the file systems supported in Hadoop.
@echo namenode -format format the DFS filesystem
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
index 36abea7..281221d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
@@ -26,8 +26,8 @@ HDFS Commands Guide
hdfs script without any arguments prints the description for all
commands.
- Usage: <<<hdfs [--config confdir] [COMMAND] [GENERIC_OPTIONS]
- [COMMAND_OPTIONS]>>>
+ Usage: <<<hdfs [--config confdir] [--loglevel loglevel] [COMMAND]
+ [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
Hadoop has an option parsing framework that employs parsing generic options
as well as running classes.
@@ -38,6 +38,10 @@ HDFS Commands Guide
| <<<--config confdir>>>| Overwrites the default Configuration directory.
| | Default is <<<${HADOOP_HOME}/conf>>>.
*-----------------------+---------------+
+| <<<--loglevel loglevel>>>| Overwrites the log level. Valid log levels are
+| | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE.
+| | Default is INFO.
+*-----------------------+---------------+
| GENERIC_OPTIONS | The common set of options supported by multiple
| | commands. Full list is
| | {{{../hadoop-common/CommandsManual.html#Generic_Options}here}}.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index 2163cad..1fa5d44 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -17,7 +17,8 @@
function hadoop_usage
{
- echo "Usage: mapred [--config confdir] [--daemon (start|stop|status)] COMMAND"
+ echo "Usage: mapred [--config confdir] [--daemon (start|stop|status)]"
+ echo " [--loglevel loglevel] COMMAND"
echo " where COMMAND is one of:"
echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-mapreduce-project/bin/mapred-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh
index cc62b9e..39baf4e 100644
--- a/hadoop-mapreduce-project/bin/mapred-config.sh
+++ b/hadoop-mapreduce-project/bin/mapred-config.sh
@@ -49,7 +49,7 @@ function hadoop_subproject_init
HADOOP_PID_DIR="${HADOOP_MAPRED_PID_DIR:-$HADOOP_PID_DIR}"
HADOOP_MAPRED_PID_DIR="${HADOOP_PID_DIR}"
- HADOOP_ROOT_LOGGER="${HADOOP_MAPRED_ROOT_LOGGER:-INFO,console}"
+ HADOOP_ROOT_LOGGER="${HADOOP_MAPRED_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
HADOOP_MAPRED_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_PREFIX}"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-mapreduce-project/bin/mapred.cmd
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred.cmd b/hadoop-mapreduce-project/bin/mapred.cmd
index 8457ba0..bb59c03 100644
--- a/hadoop-mapreduce-project/bin/mapred.cmd
+++ b/hadoop-mapreduce-project/bin/mapred.cmd
@@ -37,6 +37,11 @@ if "%1" == "--config" (
shift
)
+if "%1" == "--loglevel" (
+ shift
+ shift
+)
+
:main
if exist %MAPRED_CONF_DIR%\mapred-env.cmd (
call %MAPRED_CONF_DIR%\mapred-env.cmd
@@ -162,6 +167,10 @@ goto :eof
shift
shift
)
+ if "%1" == "--loglevel" (
+ shift
+ shift
+ )
shift
set _mapredarguments=
:MakeCmdArgsLoop
@@ -184,7 +193,7 @@ goto :eof
goto print_usage
:print_usage
- @echo Usage: mapred [--config confdir] COMMAND
+ @echo Usage: mapred [--config confdir] [--loglevel loglevel] COMMAND
@echo where COMMAND is one of:
@echo job manipulate MapReduce jobs
@echo queue get information regarding JobQueues
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-mapreduce-project/conf/mapred-env.cmd
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/conf/mapred-env.cmd b/hadoop-mapreduce-project/conf/mapred-env.cmd
index 610d593..f27943f 100644
--- a/hadoop-mapreduce-project/conf/mapred-env.cmd
+++ b/hadoop-mapreduce-project/conf/mapred-env.cmd
@@ -16,5 +16,5 @@
set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
-set HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+set HADOOP_MAPRED_ROOT_LOGGER=%HADOOP_LOGLEVEL%,RFA
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm
index 9edc41d..e011563 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm
@@ -28,7 +28,7 @@ MapReduce Commands Guide
MapReduce commands are invoked by the <<<bin/mapred>>> script. Running the
script without any arguments prints the description for all commands.
- Usage: <<<mapred [--config confdir] COMMAND>>>
+ Usage: <<<mapred [--config confdir] [--loglevel loglevel] COMMAND>>>
MapReduce has an option parsing framework that employs parsing generic
options as well as running classes.
@@ -39,6 +39,9 @@ MapReduce Commands Guide
| --config confdir | Overwrites the default Configuration directory. Default
| | is $\{HADOOP_PREFIX\}/conf.
*-------------------------+---------------------------------------------------+
+| --loglevel loglevel | Overwrites the log level. Valid log levels are FATAL,
+| | ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO.
+*-------------------------+---------------------------------------------------+
| COMMAND COMMAND_OPTIONS | Various commands with their options are described
| | in the following sections. The commands have been
| | grouped into {{User Commands}} and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 0d83674..90e5527 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -17,7 +17,8 @@
function hadoop_usage
{
- echo "Usage: yarn [--config confdir] [--daemon (start|stop|status)] COMMAND"
+ echo "Usage: yarn [--config confdir] [--daemon (start|stop|status)]"
+ echo " [--loglevel loglevel] COMMAND"
echo "where COMMAND is one of:"
echo " application prints application(s) report/kill application"
echo " applicationattempt prints applicationattempt(s) report"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
index 372e8ef..7fd7ead 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
@@ -58,7 +58,7 @@ function hadoop_subproject_init
HADOOP_PID_DIR="${YARN_PID_DIR:-$HADOOP_PID_DIR}"
YARN_PID_DIR="${HADOOP_PID_DIR}"
- HADOOP_ROOT_LOGGER="${YARN_ROOT_LOGGER:-INFO,console}"
+ HADOOP_ROOT_LOGGER="${YARN_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
YARN_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}"
HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_PREFIX}"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 7b7fdc4..b3286e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -64,6 +64,10 @@ if "%1" == "--config" (
shift
shift
)
+if "%1" == "--loglevel" (
+ shift
+ shift
+)
:main
if exist %YARN_CONF_DIR%\yarn-env.cmd (
@@ -273,6 +277,10 @@ goto :eof
shift
shift
)
+ if "%1" == "--loglevel" (
+ shift
+ shift
+ )
if [%2] == [] goto :eof
shift
set _yarnarguments=
@@ -291,7 +299,7 @@ goto :eof
goto :eof
:print_usage
- @echo Usage: yarn [--config confdir] COMMAND
+ @echo Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND
@echo where COMMAND is one of:
@echo resourcemanager run the ResourceManager
@echo nodemanager run a nodemanager on each slave
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd
index 3329f8f..d863c1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd
@@ -42,7 +42,7 @@ if not defined YARN_POLICYFILE (
)
if not defined YARN_ROOT_LOGGER (
- set YARN_ROOT_LOGGER=INFO,console
+ set YARN_ROOT_LOGGER=%HADOOP_LOGLEVEL%,console
)
set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.dir=%YARN_LOG_DIR%
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0abb973f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
index d91c513..cf41271 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
@@ -26,7 +26,7 @@ Yarn Commands
without any arguments prints the description for all commands.
------
-Usage: yarn [--config confdir] COMMAND
+Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND
------
Yarn has an option parsing framework that employs parsing generic options as
@@ -38,6 +38,9 @@ Usage: yarn [--config confdir] COMMAND
| --config confdir | Overwrites the default Configuration directory. Default
| | is $\{HADOOP_PREFIX\}/conf.
*---------------+--------------+
+| --loglevel loglevel | Overwrites the log level. Valid log levels are FATAL,
+| | ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO.
+*---------------+--------------+
| COMMAND COMMAND_OPTIONS | Various commands with their options are described
| | in the following sections. The commands have been
| | grouped into {{User Commands}} and
[29/45] hadoop git commit: Revert "YARN-2603. ApplicationConstants
missing HADOOP_MAPRED_HOME (Ray Chiang via aw)"
Posted by vi...@apache.org.
Revert "YARN-2603. ApplicationConstants missing HADOOP_MAPRED_HOME (Ray Chiang via aw)"
This reverts commit b0a9cd37ab9ef45ed9a71b04f1634fa54dd4c438.
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ae9780e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ae9780e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ae9780e
Branch: refs/heads/HDFS-EC
Commit: 4ae9780e6a05bfd6b93f1c871c22761ddd8b19cb
Parents: 9ff7315
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu Nov 13 13:17:11 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Nov 13 13:17:11 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ---
.../java/org/apache/hadoop/yarn/api/ApplicationConstants.java | 5 -----
2 files changed, 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae9780e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3235907..60f88ca 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -81,9 +81,6 @@ Release 2.7.0 - UNRELEASED
YARN-2713. "RM Home" link in NM should point to one of the RMs in an
HA setup. (kasha)
- YARN-2603. ApplicationConstants missing HADOOP_MAPRED_HOME (Ray Chiang via
- aw)
-
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae9780e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index 93438b4..d42cd67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -181,11 +181,6 @@ public interface ApplicationConstants {
HADOOP_HDFS_HOME("HADOOP_HDFS_HOME"),
/**
- * $HADOOP_MAPRED_HOME
- */
- HADOOP_MAPRED_HOME("HADOOP_MAPRED_HOME"),
-
- /**
* $MALLOC_ARENA_MAX
*/
MALLOC_ARENA_MAX("MALLOC_ARENA_MAX"),
[43/45] hadoop git commit: YARN-1703. Fixed ResourceManager web-proxy
to close connections correctly. Contributed by Rohith Sharma.
Posted by vi...@apache.org.
YARN-1703. Fixed ResourceManager web-proxy to close connections correctly. Contributed by Rohith Sharma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/198fb587
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/198fb587
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/198fb587
Branch: refs/heads/HDFS-EC
Commit: 198fb58776ec02397af5ad83259535b0291b745c
Parents: 9b86066
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Sat Nov 15 12:05:25 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Nov 15 12:05:25 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/198fb587/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9d3179b..b303e53 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -96,6 +96,9 @@ Release 2.7.0 - UNRELEASED
YARN-2432. RMStateStore should process the pending events before close.
(Varun Saxena via jianhe)
+ YARN-1703. Fixed ResourceManager web-proxy to close connections correctly.
+ (Rohith Sharma via vinodkv)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/198fb587/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
index 19ae9dc..31457a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
@@ -166,6 +166,7 @@ public class WebAppProxyServlet extends HttpServlet {
}
config.setLocalAddress(localAddress);
HttpMethod method = new GetMethod(uri.getEscapedURI());
+ method.setRequestHeader("Connection","close");
@SuppressWarnings("unchecked")
Enumeration<String> names = req.getHeaderNames();
while(names.hasMoreElements()) {
[24/45] hadoop git commit: HADOOP-11298. slaves.sh and stop-all.sh
are missing slashes (aw)
Posted by vi...@apache.org.
HADOOP-11298. slaves.sh and stop-all.sh are missing slashes (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b650e61a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b650e61a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b650e61a
Branch: refs/heads/HDFS-EC
Commit: b650e61a7f5bd27be850b4ec940516571b4c6b3d
Parents: ba879a5
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Nov 13 12:01:38 2014 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Nov 13 12:01:38 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
hadoop-common-project/hadoop-common/src/main/bin/slaves.sh | 2 +-
hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh | 2 +-
3 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b650e61a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d77d024..f3a0237 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -343,6 +343,8 @@ Trunk (Unreleased)
HADOOP-11284. Fix variable name mismatches in hadoop-functions.sh (Masatake
Iwasaki via aw)
+ HADOOP-11298. slaves.sh and stop-all.sh are missing slashes (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b650e61a/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh b/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh
index 0ffb484..a8f0660 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/slaves.sh
@@ -36,7 +36,7 @@ if [[ -n "${HADOOP_PREFIX}" ]]; then
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
else
this="${BASH_SOURCE-$0}"
- bin=$(cd -P -- "$(dirname -- "${this}")" >dev/null && pwd -P)
+ bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b650e61a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh
index a0311e4..55424d8 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh
@@ -26,7 +26,7 @@ if [[ -n "${HADOOP_PREFIX}" ]]; then
DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
else
this="${BASH_SOURCE-$0}"
- bin=$(cd -P -- "$(dirname -- "${this}")" >dev/null && pwd -P)
+ bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi
[09/45] hadoop git commit: HADOOP-11025. hadoop-daemons.sh should
just call hdfs directly (Masatake Iwasaki via aw)
Posted by vi...@apache.org.
HADOOP-11025. hadoop-daemons.sh should just call hdfs directly (Masatake Iwasaki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/306f0c9c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/306f0c9c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/306f0c9c
Branch: refs/heads/HDFS-EC
Commit: 306f0c9cea2d35f3f9ebf25ad6af99723e4c7288
Parents: 782abbb
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Nov 12 15:46:58 2014 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Nov 12 15:46:58 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/main/bin/hadoop-daemons.sh | 19 ++++++++++++++-----
.../hadoop-common/src/main/conf/hadoop-env.sh | 3 ++-
3 files changed, 19 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/306f0c9c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 480737b..59773ae 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -135,6 +135,9 @@ Trunk (Unreleased)
HADOOP-11231. Remove dead code in ServletUtil. (Li Lu via wheat9)
+ HADOOP-11025. hadoop-daemons.sh should just call hdfs directly (Masatake
+ Iwasaki via aw)
+
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/306f0c9c/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
index db06612..20d7e4a 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
@@ -36,10 +36,10 @@ fi
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
- . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
exit 1
fi
@@ -47,5 +47,14 @@ if [[ $# = 0 ]]; then
hadoop_exit_with_usage 1
fi
-hadoop_connect_to_hosts "${bin}/hadoop-daemon.sh" \
---config "${HADOOP_CONF_DIR}" "$@"
+daemonmode=$1
+shift
+
+if [[ -z "${HADOOP_HDFS_HOME}" ]]; then
+ hdfsscript="${HADOOP_PREFIX}/bin/hdfs"
+else
+ hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs"
+fi
+
+hadoop_connect_to_hosts "$hdfsscript" \
+ --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/306f0c9c/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index b0963af..af8e544 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -175,7 +175,8 @@ esac
# This sets the hadoop.root.logger property
# export HADOOP_ROOT_LOGGER=INFO,console
-# Default log level for daemons spawned explicitly by hadoop-daemon.sh
+# Default log level for daemons spawned explicitly by
+# --daemon option of hadoop, hdfs, mapred and yarn command.
# This sets the hadoop.root.logger property
# export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA
[20/45] hadoop git commit: YARN-2853. Merging to branch-2.6 for
hadoop-2.6.0-rc1.
Posted by vi...@apache.org.
YARN-2853. Merging to branch-2.6 for hadoop-2.6.0-rc1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d648e60e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d648e60e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d648e60e
Branch: refs/heads/HDFS-EC
Commit: d648e60ebab7f1942dba92e9cd2cb62b8d70419b
Parents: 3651fe1
Author: Arun C. Murthy <ac...@apache.org>
Authored: Thu Nov 13 10:04:45 2014 -0800
Committer: Arun C. Murthy <ac...@apache.org>
Committed: Thu Nov 13 10:04:45 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d648e60e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 32e9e19..2179faf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -87,9 +87,6 @@ Release 2.7.0 - UNRELEASED
YARN-2603. ApplicationConstants missing HADOOP_MAPRED_HOME (Ray Chiang via
aw)
- YARN-2853. Fixed a bug in ResourceManager causing apps to hang when the user
- kill request races with ApplicationMaster finish. (Jian He via vinodkv)
-
Release 2.6.0 - 2014-11-15
INCOMPATIBLE CHANGES
@@ -962,6 +959,9 @@ Release 2.6.0 - 2014-11-15
reacquireContainer() that interrupted by NodeManager restart. (Junping Du
via jlowe)
+ YARN-2853. Fixed a bug in ResourceManager causing apps to hang when the user
+ kill request races with ApplicationMaster finish. (Jian He via vinodkv)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
[10/45] hadoop git commit: HADOOP-11284. Fix variable name mismatches
in hadoop-functions.sh (Masatake Iwasaki via aw)
Posted by vi...@apache.org.
HADOOP-11284. Fix variable name mismatches in hadoop-functions.sh (Masatake Iwasaki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7150a1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7150a1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7150a1b
Branch: refs/heads/HDFS-EC
Commit: d7150a1bc2fdf87bedb112ffbd1c5cab7f14d1c3
Parents: 306f0c9
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Nov 12 15:58:51 2014 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Nov 12 15:58:51 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/main/bin/hadoop-functions.sh | 14 +++++++-------
2 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7150a1b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 59773ae..f8fca16 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -340,6 +340,9 @@ Trunk (Unreleased)
HADOOP-11248. Add hadoop configuration to disable Azure Filesystem metrics
collection. (Shanyu Zhao via cnauroth)
+ HADOOP-11284. Fix variable name mismatches in hadoop-functions.sh (Masatake
+ Iwasaki via aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7150a1b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index efa42f6..52d5af9 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -837,13 +837,13 @@ function hadoop_start_daemon_wrapper
# shellcheck disable=SC2086
renice "${HADOOP_NICENESS}" $! >/dev/null 2>&1
if [[ $? -gt 0 ]]; then
- hadoop_error "ERROR: Cannot set priority of ${daemoname} process $!"
+ hadoop_error "ERROR: Cannot set priority of ${daemonname} process $!"
fi
# shellcheck disable=SC2086
disown %+ >/dev/null 2>&1
if [[ $? -gt 0 ]]; then
- hadoop_error "ERROR: Cannot disconnect ${daemoname} process $!"
+ hadoop_error "ERROR: Cannot disconnect ${daemonname} process $!"
fi
sleep 1
@@ -898,7 +898,7 @@ function hadoop_start_secure_daemon
#shellcheck disable=SC2086
echo $$ > "${privpidfile}" 2>/dev/null
if [[ $? -gt 0 ]]; then
- hadoop_error "ERROR: Cannot write ${daemoname} pid ${privpidfile}."
+ hadoop_error "ERROR: Cannot write ${daemonname} pid ${privpidfile}."
fi
exec "${jsvc}" \
@@ -954,7 +954,7 @@ function hadoop_start_secure_daemon_wrapper
# so let's wait for the fork to finish
# before overriding with the daemonized pid
(( counter=0 ))
- while [[ ! -f ${pidfile} && ${counter} -le 5 ]]; do
+ while [[ ! -f ${daemonpidfile} && ${counter} -le 5 ]]; do
sleep 1
(( counter++ ))
done
@@ -963,7 +963,7 @@ function hadoop_start_secure_daemon_wrapper
#shellcheck disable=SC2086
echo $! > "${jsvcpidfile}" 2>/dev/null
if [[ $? -gt 0 ]]; then
- hadoop_error "ERROR: Cannot write ${daemonname} pid ${pidfile}."
+ hadoop_error "ERROR: Cannot write ${daemonname} pid ${daemonpidfile}."
fi
sleep 1
@@ -1037,8 +1037,8 @@ function hadoop_daemon_handler
local daemonmode=$1
local daemonname=$2
local class=$3
- local pidfile=$4
- local outfile=$5
+ local daemon_pidfile=$4
+ local daemon_outfile=$5
shift 5
case ${daemonmode} in
[35/45] hadoop git commit: HADOOP-11257. Update "hadoop jar"
documentation to warn against using it for launching yarn jars (iwasakims via
cmccabe)
Posted by vi...@apache.org.
HADOOP-11257. Update "hadoop jar" documentation to warn against using it for launching yarn jars (iwasakims via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10c98aed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10c98aed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10c98aed
Branch: refs/heads/HDFS-EC
Commit: 10c98aed38430df5a18899b098c34f5874a798d4
Parents: f2fe8a8
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Fri Nov 14 12:53:09 2014 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Fri Nov 14 12:53:09 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/src/main/bin/hadoop | 5 +++++
hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd | 7 +++++++
.../hadoop-common/src/site/apt/CommandsManual.apt.vm | 3 +++
3 files changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10c98aed/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index eccd322..9df2c7d 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -33,6 +33,8 @@ function hadoop_usage()
echo " copy file or directories recursively"
echo " fs run a generic filesystem user client"
echo " jar <jar> run a jar file"
+ echo " note: please use \"yarn jar\" to launch"
+ echo " YARN applications, not this command."
echo " jnipath prints the java.library.path"
echo " key manage keys via the KeyProvider"
echo " trace view and modify Hadoop tracing settings"
@@ -150,6 +152,9 @@ case ${COMMAND} in
CLASS=org.apache.hadoop.fs.FsShell
;;
jar)
+ if [[ -n "${YARN_OPTS}" ]] || [[ -n "${YARN_CLIENT_OPTS}" ]]; then
+ hadoop_error "WARNING: Use \"yarn jar\" to launch YARN applications."
+ fi
CLASS=org.apache.hadoop.util.RunJar
;;
jnipath)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10c98aed/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
index fc2e9b7..c537539 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
@@ -177,6 +177,11 @@ call :updatepath %HADOOP_BIN_PATH%
goto :eof
:version
+ if defined YARN_OPTS (
+ @echo WARNING: Use "yarn jar" to launch YARN applications.
+ ) else if defined YARN_CLIENT_OPTS (
+ @echo WARNING: Use "yarn jar" to launch YARN applications.
+ )
set CLASS=org.apache.hadoop.util.VersionInfo
goto :eof
@@ -261,6 +266,8 @@ call :updatepath %HADOOP_BIN_PATH%
goto :MakeCmdArgsLoop
:EndLoop
set hadoop-command-arguments=%_arguments%
+ @echo note: please use "yarn jar" to launch
+ @echo YARN applications, not this command.
goto :eof
:print_usage
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10c98aed/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
index 7771aa1..6d2fd5e 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
@@ -177,6 +177,9 @@ User Commands
Word count example is also run using jar command. It can be referred from
Wordcount example
+ Use {{{../../hadoop-yarn/hadoop-yarn-site/YarnCommands.html#jar}<<<yarn jar>>>}}
+ to launch YARN applications instead.
+
* <<<job>>>
Deprecated. Use
[39/45] hadoop git commit: YARN-2811. In Fair Scheduler,
reservation fulfillments shouldn't ignore max share (Siqi Li via
Sandy Ryza)
Posted by vi...@apache.org.
YARN-2811. In Fair Scheduler, reservation fulfillments shouldn't ignore max share (Siqi Li via Sandy Ryza)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a47f890
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a47f890
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a47f890
Branch: refs/heads/HDFS-EC
Commit: 1a47f890ba3cb22b6262f47c1f1af2990559bb89
Parents: 49c3889
Author: Sandy Ryza <sa...@cloudera.com>
Authored: Fri Nov 14 15:18:56 2014 -0800
Committer: Sandy Ryza <sa...@cloudera.com>
Committed: Fri Nov 14 15:18:56 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/fair/FairScheduler.java | 18 +++-
.../scheduler/fair/TestFairScheduler.java | 88 +++++++++++++++++++-
3 files changed, 103 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a47f890/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1f8af55..cfd9459 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -225,6 +225,9 @@ Release 2.6.0 - 2014-11-18
YARN-2505. Supported get/add/remove/change labels in RM REST API. (Craig Welch
via zjshen)
+ YARN-2811. In Fair Scheduler, reservation fulfillments shouldn't ignore max
+ share (Siqi Li via Sandy Ryza)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a47f890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 3fc3019..94fb849 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1029,7 +1029,10 @@ public class FairScheduler extends
FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
if (reservedAppSchedulable != null) {
Priority reservedPriority = node.getReservedContainer().getReservedPriority();
- if (!reservedAppSchedulable.hasContainerForNode(reservedPriority, node)) {
+ FSQueue queue = reservedAppSchedulable.getQueue();
+
+ if (!reservedAppSchedulable.hasContainerForNode(reservedPriority, node)
+ || !fitInMaxShare(queue)) {
// Don't hold the reservation if app can no longer use it
LOG.info("Releasing reservation that cannot be satisfied for application "
+ reservedAppSchedulable.getApplicationAttemptId()
@@ -1043,7 +1046,6 @@ public class FairScheduler extends
+ reservedAppSchedulable.getApplicationAttemptId()
+ " on node: " + node);
}
-
node.getReservedAppSchedulable().assignReservedContainer(node);
}
}
@@ -1065,6 +1067,18 @@ public class FairScheduler extends
updateRootQueueMetrics();
}
+ private boolean fitInMaxShare(FSQueue queue) {
+ if (Resources.fitsIn(queue.getResourceUsage(), queue.getMaxShare())) {
+ return false;
+ }
+
+ FSQueue parentQueue = queue.getParent();
+ if (parentQueue != null) {
+ return fitInMaxShare(parentQueue);
+ }
+ return true;
+ }
+
public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) {
return super.getApplicationAttempt(appAttemptId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a47f890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 61cbdc1..f223458 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -722,6 +722,85 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
+ @Test (timeout = 5000)
+ public void testContainerReservationNotExceedingQueueMax() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"root\">");
+ out.println("<queue name=\"queue1\">");
+ out.println("<minResources>1024mb,5vcores</minResources>");
+ out.println("<maxResources>2048mb,10vcores</maxResources>");
+ out.println("</queue>");
+ out.println("<queue name=\"queue2\">");
+ out.println("<minResources>1024mb,5vcores</minResources>");
+ out.println("<maxResources>2048mb,10vcores</maxResources>");
+ out.println("</queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node
+ RMNode node1 =
+ MockNodes
+ .newNodeInfo(1, Resources.createResource(3072, 5), 1, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Queue 1 requests full capacity of the queue
+ createSchedulingRequest(2048, "queue1", "user1", 1);
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
+ scheduler.handle(updateEvent);
+
+ // Make sure queue 1 is allocated app capacity
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getResourceUsage().getMemory());
+
+ // Now queue 2 requests likewise
+ ApplicationAttemptId attId = createSchedulingRequest(1024, "queue2", "user2", 1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // Make sure queue 2 is allocated app capacity
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getResourceUsage().getMemory());
+
+ ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1", 1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+
+ // Make sure queue 1 is waiting with a reservation
+ assertEquals(1024, scheduler.getSchedulerApp(attId1)
+ .getCurrentReservation().getMemory());
+
+ // Now remove app of queue2
+ AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(
+ attId, RMAppAttemptState.FINISHED, false);
+ scheduler.update();
+ scheduler.handle(appRemovedEvent1);
+
+ // Queue should have no apps
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getResourceUsage().getMemory());
+
+ createSchedulingRequest(1024, "queue2", "user2", 1);
+ scheduler.handle(updateEvent);
+ // Make sure allocated memory of queue1 doesn't exceed its maximum
+ assertEquals(2048, scheduler.getQueueManager().getQueue("queue1").
+ getResourceUsage().getMemory());
+ //the reservation of queue1 should be reclaim
+ assertEquals(0, scheduler.getSchedulerApp(attId1).
+ getCurrentReservation().getMemory());
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getResourceUsage().getMemory());
+ }
+
@Test
public void testUserAsDefaultQueue() throws Exception {
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
@@ -2076,9 +2155,10 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.handle(updateEvent);
assertEquals(1, app.getLiveContainers().size());
- // Reserved container should still be at lower priority
+ // Reserved container should will be at higher priority,
+ // since old reservation cannot be satisfied
for (RMContainer container : app.getReservedContainers()) {
- assertEquals(2, container.getReservedPriority().getPriority());
+ assertEquals(1, container.getReservedPriority().getPriority());
}
// Complete container
@@ -2091,11 +2171,11 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.update();
scheduler.handle(updateEvent);
- // Reserved container (at lower priority) should be run
+ // Reserved container (at higher priority) should be run
Collection<RMContainer> liveContainers = app.getLiveContainers();
assertEquals(1, liveContainers.size());
for (RMContainer liveContainer : liveContainers) {
- Assert.assertEquals(2, liveContainer.getContainer().getPriority().getPriority());
+ Assert.assertEquals(1, liveContainer.getContainer().getPriority().getPriority());
}
assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
[28/45] hadoop git commit: Preparing to release hadoop-2.6.0-rc1.
Posted by vi...@apache.org.
Preparing to release hadoop-2.6.0-rc1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ff7315c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ff7315c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ff7315c
Branch: refs/heads/HDFS-EC
Commit: 9ff7315cbe9804bebcd1ec093fac7828a72ddf59
Parents: b0a41de
Author: Arun C. Murthy <ac...@apache.org>
Authored: Thu Nov 13 13:02:55 2014 -0800
Committer: Arun C. Murthy <ac...@apache.org>
Committed: Thu Nov 13 13:02:55 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +-
.../src/main/docs/releasenotes.html | 54 ++++++++++++++++++--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 2 +-
hadoop-yarn-project/CHANGES.txt | 2 +-
5 files changed, 53 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ff7315c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1a8c41d..8b9b866 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -441,7 +441,7 @@ Release 2.7.0 - UNRELEASED
HADOOP-11294. Nfs3FileAttributes should not change the values of rdev,
nlink and size in the constructor. (Brandon Li via wheat9)
-Release 2.6.0 - 2014-11-15
+Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ff7315c/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
index 85e3ae6..9880272 100644
--- a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
+++ b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
@@ -12,6 +12,22 @@ These release notes include new developer and user-facing incompatibilities, fea
<a name="changes"/>
<h2>Changes since Hadoop 2.5.1</h2>
<ul>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2853">YARN-2853</a>.
+ Major bug reported by Jian He and fixed by Jian He <br>
+ <b>Killing app may hang while AM is unregistering</b><br>
+ <blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2846">YARN-2846</a>.
+ Blocker bug reported by Junping Du and fixed by Junping Du (nodemanager)<br>
+ <b>Incorrect persist exit code for running containers in reacquireContainer() that interrupted by NodeManager restart.</b><br>
+ <blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2843">YARN-2843</a>.
+ Major sub-task reported by Sushmitha Sreenivasan and fixed by Wangda Tan (resourcemanager)<br>
+ <b>NodeLabels manager should trim all inputs for hosts and labels</b><br>
+ <blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2841">YARN-2841</a>.
+ Critical sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
+ <b>RMProxy should retry EOFException </b><br>
+ <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/YARN-2834">YARN-2834</a>.
Blocker bug reported by Yesha Vora and fixed by Jian He <br>
<b>Resource manager crashed with Null Pointer Exception</b><br>
@@ -25,7 +41,7 @@ These release notes include new developer and user-facing incompatibilities, fea
<b>Fix bugs of yarn queue CLI</b><br>
<blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/YARN-2826">YARN-2826</a>.
- Critical bug reported by sidharta seethana and fixed by Wangda Tan <br>
+ Critical bug reported by Sidharta Seethana and fixed by Wangda Tan <br>
<b>User-Group mappings not updated by RM when a user is removed from a group.</b><br>
<blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/YARN-2825">YARN-2825</a>.
@@ -80,6 +96,10 @@ These release notes include new developer and user-facing incompatibilities, fea
Major sub-task reported by Phil D'Amore and fixed by Wangda Tan (resourcemanager)<br>
<b>Resource Manager fails startup with HDFS label storage and secure cluster</b><br>
<blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2794">YARN-2794</a>.
+ Major bug reported by Jian He and fixed by Jian He <br>
+ <b>Fix log msgs about distributing system-credentials </b><br>
+ <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/YARN-2790">YARN-2790</a>.
Critical bug reported by Tassapol Athiapinya and fixed by Jian He (nodemanager)<br>
<b>NM can't aggregate logs past HDFS delegation token expiry.</b><br>
@@ -304,6 +324,10 @@ These release notes include new developer and user-facing incompatibilities, fea
Major sub-task reported by Craig Welch and fixed by Craig Welch <br>
<b>Recalculate headroom more frequently to keep it accurate</b><br>
<blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2635">YARN-2635</a>.
+ Major bug reported by Wei Yan and fixed by Wei Yan <br>
+ <b>TestRM, TestRMRestart, TestClientToAMTokens should run with both CS and FS</b><br>
+ <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/YARN-2632">YARN-2632</a>.
Blocker sub-task reported by Junping Du and fixed by Junping Du (nodemanager)<br>
<b>Document NM Restart feature</b><br>
@@ -1000,6 +1024,10 @@ These release notes include new developer and user-facing incompatibilities, fea
Major sub-task reported by Remus Rusanu and fixed by Remus Rusanu (nodemanager)<br>
<b>Implement secure Windows Container Executor</b><br>
<blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-1964">YARN-1964</a>.
+ Major new feature reported by Arun C Murthy and fixed by Abin Shahab <br>
+ <b>Create Docker analog of the LinuxContainerExecutor in YARN</b><br>
+ <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/YARN-1959">YARN-1959</a>.
Major bug reported by Sandy Ryza and fixed by Anubhav Dhoot <br>
<b>Fix headroom calculation in FairScheduler</b><br>
@@ -1128,6 +1156,10 @@ These release notes include new developer and user-facing incompatibilities, fea
Major sub-task reported by Ravi Gummadi and fixed by Varun Vasudev (nodemanager)<br>
<b>NodeManager should identify failed disks becoming good again</b><br>
<blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6156">MAPREDUCE-6156</a>.
+ Blocker bug reported by Sidharta Seethana and fixed by Junping Du <br>
+ <b>Fetcher - connect() doesn't handle connection refused correctly </b><br>
+ <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/MAPREDUCE-6142">MAPREDUCE-6142</a>.
Critical sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
<b>Test failure in TestJobHistoryEventHandler and TestMRTimelineEventHandling</b><br>
@@ -1404,6 +1436,18 @@ These release notes include new developer and user-facing incompatibilities, fea
Minor improvement reported by Koji Noguchi and fixed by Akira AJISAKA (documentation , harchive)<br>
<b>harchive: Document how to unarchive</b><br>
<blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/HDFS-7391">HDFS-7391</a>.
+ Blocker bug reported by Robert Kanter and fixed by Robert Kanter (webhdfs)<br>
+ <b>Renable SSLv2Hello in HttpFS</b><br>
+ <blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/HDFS-7387">HDFS-7387</a>.
+ Critical bug reported by Brandon Li and fixed by Brandon Li (nfs)<br>
+ <b>NFS may only do partial commit due to a race between COMMIT and write</b><br>
+ <blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/HDFS-7385">HDFS-7385</a>.
+ Blocker bug reported by jiangyu and fixed by jiangyu (namenode)<br>
+ <b>ThreadLocal used in FSEditLog class causes FSImage permission mess up</b><br>
+ <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/HDFS-7383">HDFS-7383</a>.
Major bug reported by Tsz Wo Nicholas Sze and fixed by Tsz Wo Nicholas Sze (datanode)<br>
<b>DataNode.requestShortCircuitFdsForRead may throw NullPointerException</b><br>
@@ -1952,10 +1996,6 @@ These release notes include new developer and user-facing incompatibilities, fea
Trivial bug reported by Ray Chiang and fixed by Ray Chiang <br>
<b>Fix typos in log messages</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/HDFS-6938">HDFS-6938</a>.
- Trivial bug reported by Charles Lamb and fixed by Charles Lamb (namenode)<br>
- <b>Cleanup javac warnings in FSNamesystem</b><br>
- <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/HDFS-6934">HDFS-6934</a>.
Major sub-task reported by Arpit Agarwal and fixed by Chris Nauroth (datanode , hdfs-client)<br>
<b>Move checksum computation off the hot path when writing to RAM disk</b><br>
@@ -3489,6 +3529,10 @@ with no args would throw an NPE.</blockquote></li>
Major bug reported by Allan Yan and fixed by Allan Yan (fs)<br>
<b>FsShell's Text command does not read avro data files stored on HDFS</b><br>
<blockquote></blockquote></li>
+<li> <a href="https://issues.apache.org/jira/browse/HADOOP-9576">HADOOP-9576</a>.
+ Major bug reported by Jian He and fixed by Steve Loughran <br>
+ <b>Make NetUtils.wrapException throw EOFException instead of wrapping it as IOException</b><br>
+ <blockquote></blockquote></li>
<li> <a href="https://issues.apache.org/jira/browse/HADOOP-9540">HADOOP-9540</a>.
Minor improvement reported by Hari and fixed by (fs/s3 , test)<br>
<b>Expose the InMemoryS3 and S3N FilesystemStores implementations for Unit testing.</b><br>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ff7315c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 998a395..e0ef064 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -424,7 +424,7 @@ Release 2.7.0 - UNRELEASED
HDFS-7358. Clients may get stuck waiting when using ByteArrayManager.
(szetszwo)
-Release 2.6.0 - 2014-11-15
+Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ff7315c/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index c0105e6..a8ca13f 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -236,7 +236,7 @@ Release 2.7.0 - UNRELEASED
BUG FIXES
-Release 2.6.0 - 2014-11-15
+Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ff7315c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8f679fd..3235907 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -84,7 +84,7 @@ Release 2.7.0 - UNRELEASED
YARN-2603. ApplicationConstants missing HADOOP_MAPRED_HOME (Ray Chiang via
aw)
-Release 2.6.0 - 2014-11-15
+Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
[41/45] hadoop git commit: HDFS-7386. Replace check "port number <
1024" with shared isPrivilegedPort method. Contributed by Yongjun Zhang.
Posted by vi...@apache.org.
HDFS-7386. Replace check "port number < 1024" with shared isPrivilegedPort method. Contributed by Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1925e2a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1925e2a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1925e2a4
Branch: refs/heads/HDFS-EC
Commit: 1925e2a4ae78ef4178393848b4d1d71b0f4a4709
Parents: 4fb96db
Author: cnauroth <cn...@apache.org>
Authored: Fri Nov 14 16:43:09 2014 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Fri Nov 14 16:43:09 2014 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/security/SecurityUtil.java | 15 +++++++++++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../datatransfer/sasl/SaslDataTransferClient.java | 3 ++-
.../datatransfer/sasl/SaslDataTransferServer.java | 3 ++-
.../hdfs/server/datanode/SecureDataNodeStarter.java | 3 ++-
5 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1925e2a4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
index 27870c3..7cbee26 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java
@@ -621,4 +621,19 @@ public class SecurityUtil {
conf.set(HADOOP_SECURITY_AUTHENTICATION,
authenticationMethod.toString().toLowerCase(Locale.ENGLISH));
}
+
+ /*
+ * Check if a given port is privileged.
+ * The ports with number smaller than 1024 are treated as privileged ports in
+ * unix/linux system. For other operating systems, use this method with care.
+ * For example, Windows doesn't have the concept of privileged ports.
+ * However, it may be used at Windows client to check port of linux server.
+ *
+ * @param port the port number
+ * @return true for privileged ports, false otherwise
+ *
+ */
+ public static boolean isPrivilegedPort(final int port) {
+ return port < 1024;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1925e2a4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 957d7ae..2917ea9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -356,6 +356,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7375. Move FSClusterStats to o.a.h.h.hdfs.server.blockmanagement.
(wheat9)
+ HDFS-7386. Replace check "port number < 1024" with shared isPrivilegedPort
+ method. (Yongjun Zhang via cnauroth)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1925e2a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
index 98cdcfe..00b131f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferClient.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.security.SaslPropertiesResolver;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.slf4j.Logger;
@@ -245,7 +246,7 @@ public class SaslDataTransferClient {
"SASL client skipping handshake in unsecured configuration for "
+ "addr = {}, datanodeId = {}", addr, datanodeId);
return null;
- } else if (datanodeId.getXferPort() < 1024) {
+ } else if (SecurityUtil.isPrivilegedPort(datanodeId.getXferPort())) {
LOG.debug(
"SASL client skipping handshake in secured configuration with "
+ "privileged port for addr = {}, datanodeId = {}", addr, datanodeId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1925e2a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
index 3fa7727..1d2b30b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/SaslDataTransferServer.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockPoolTokenSecretManager;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.server.datanode.DNConf;
import org.apache.hadoop.security.SaslPropertiesResolver;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,7 +113,7 @@ public class SaslDataTransferServer {
"SASL server skipping handshake in unsecured configuration for "
+ "peer = {}, datanodeId = {}", peer, datanodeId);
return new IOStreamPair(underlyingIn, underlyingOut);
- } else if (xferPort < 1024) {
+ } else if (SecurityUtil.isPrivilegedPort(xferPort)) {
LOG.debug(
"SASL server skipping handshake in secured configuration for "
+ "peer = {}, datanodeId = {}", peer, datanodeId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1925e2a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
index 477b7f6..f0f83e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.mortbay.jetty.Connector;
@@ -110,7 +111,7 @@ public class SecureDataNodeStarter implements Daemon {
+ ss.getLocalPort());
}
- if (ss.getLocalPort() > 1023 && isSecure) {
+ if (!SecurityUtil.isPrivilegedPort(ss.getLocalPort()) && isSecure) {
throw new RuntimeException(
"Cannot start secure datanode with unprivileged RPC ports");
}
[40/45] hadoop git commit: HADOOP-11307. create-release script should
run git clean first (kasha)
Posted by vi...@apache.org.
HADOOP-11307. create-release script should run git clean first (kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4fb96dbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4fb96dbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4fb96dbe
Branch: refs/heads/HDFS-EC
Commit: 4fb96dbe3bbfd3ae88b41c816e15acee04518851
Parents: 1a47f89
Author: Karthik Kambatla <ka...@apache.org>
Authored: Fri Nov 14 15:35:50 2014 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Fri Nov 14 15:35:50 2014 -0800
----------------------------------------------------------------------
dev-support/create-release.sh | 6 ++++++
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fb96dbe/dev-support/create-release.sh
----------------------------------------------------------------------
diff --git a/dev-support/create-release.sh b/dev-support/create-release.sh
index 447fbdd..7a3dd70 100755
--- a/dev-support/create-release.sh
+++ b/dev-support/create-release.sh
@@ -47,6 +47,9 @@ RC_LABEL=$1
# Extract Hadoop version from POM
HADOOP_VERSION=`cat pom.xml | grep "<version>" | head -1 | sed 's|^ *<version>||' | sed 's|</version>.*$||'`
+# Setup git
+GIT=${GIT:-git}
+
echo
echo "*****************************************************************"
echo
@@ -70,6 +73,9 @@ fi
ARTIFACTS_DIR="target/artifacts"
+# git clean to clear any remnants from previous build
+run ${GIT} clean -xdf
+
# mvn clean for sanity
run ${MVN} clean
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fb96dbe/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8b9b866..bce342b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1122,6 +1122,7 @@ Release 2.6.0 - 2014-11-18
HADOOP-9576. Changed NetUtils#wrapException to throw EOFException instead
of wrapping it as IOException. (Steve Loughran via jianhe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
@@ -1138,6 +1139,7 @@ Release 2.5.2 - 2014-11-10
HADOOP-11260. Patch up Jetty to disable SSLv3. (Mike Yoder via kasha)
+ HADOOP-11307. create-release script should run git clean first (kasha)
Release 2.5.1 - 2014-09-05
[12/45] hadoop git commit: HADOOP-11217. (Addendum to allow
SSLv2Hello) Disable SSLv3 in KMS. (Robert Kanter via kasha)
Posted by vi...@apache.org.
HADOOP-11217. (Addendum to allow SSLv2Hello) Disable SSLv3 in KMS. (Robert Kanter via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87818ef4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87818ef4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87818ef4
Branch: refs/heads/HDFS-EC
Commit: 87818ef4e7c1a49dc3b0d6766cceb3499b07b837
Parents: b0a9cd3
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Nov 12 18:39:03 2014 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Nov 12 18:39:03 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87818ef4/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml b/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml
index 531ad15..5215ded 100644
--- a/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/tomcat/ssl-server.xml
@@ -70,7 +70,7 @@
described in the APR documentation -->
<Connector port="${kms.http.port}" protocol="HTTP/1.1" SSLEnabled="true"
maxThreads="${kms.max.threads}" scheme="https" secure="true"
- clientAuth="false" sslEnabledProtocols="TLSv1"
+ clientAuth="false" sslEnabledProtocols="TLSv1,SSLv2Hello"
keystoreFile="${kms.ssl.keystore.file}"
keystorePass="${kms.ssl.keystore.pass}"/>
[31/45] hadoop git commit: HDFS-7395. BlockIdManager#clear() bails
out when resetting the GenerationStampV1Limit. Contributed by Haohui Mai.
Posted by vi...@apache.org.
HDFS-7395. BlockIdManager#clear() bails out when resetting the GenerationStampV1Limit. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a2e5cbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a2e5cbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a2e5cbc
Branch: refs/heads/HDFS-EC
Commit: 1a2e5cbc4dbed527fdbefc09abc1faaacf3da285
Parents: 3648cb5
Author: Haohui Mai <wh...@apache.org>
Authored: Thu Nov 13 14:43:03 2014 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Thu Nov 13 14:43:03 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a2e5cbc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e0ef064..957d7ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -424,6 +424,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7358. Clients may get stuck waiting when using ByteArrayManager.
(szetszwo)
+ HDFS-7395. BlockIdManager#clear() bails out when resetting the
+ GenerationStampV1Limit. (wheat9)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a2e5cbc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
index 8f547f1..1c69203 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
@@ -203,6 +203,6 @@ public class BlockIdManager {
generationStampV2.setCurrentValue(GenerationStamp.LAST_RESERVED_STAMP);
getBlockIdGenerator().setCurrentValue(SequentialBlockIdGenerator
.LAST_RESERVED_BLOCK_ID);
- setGenerationStampV1Limit(GenerationStamp.GRANDFATHER_GENERATION_STAMP);
+ generationStampV1Limit = GenerationStamp.GRANDFATHER_GENERATION_STAMP;
}
}
\ No newline at end of file
[33/45] hadoop git commit: MAPREDUCE-5918. LineRecordReader can
return the same decompressor to CodecPool multiple times (Sergey Murylev via
raviprak)
Posted by vi...@apache.org.
MAPREDUCE-5918. LineRecordReader can return the same decompressor to CodecPool multiple times (Sergey Murylev via raviprak)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a1dcce8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a1dcce8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a1dcce8
Branch: refs/heads/HDFS-EC
Commit: 1a1dcce827d8747a5629151afa335598fbc94f9e
Parents: d005404
Author: Ravi Prakash <ra...@apache.org>
Authored: Fri Nov 14 03:45:53 2014 -0800
Committer: Ravi Prakash <ra...@apache.org>
Committed: Fri Nov 14 03:45:53 2014 -0800
----------------------------------------------------------------------
.../apache/hadoop/io/compress/CodecPool.java | 54 +++++++++++---------
.../hadoop/io/compress/TestCodecPool.java | 47 ++++++++++++++---
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../apache/hadoop/mapred/LineRecordReader.java | 1 +
.../mapreduce/lib/input/LineRecordReader.java | 1 +
.../hadoop/mapred/TestLineRecordReader.java | 37 ++++++++++++++
.../lib/input/TestLineRecordReader.java | 38 ++++++++++++++
7 files changed, 150 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a1dcce8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
index 11d88f1..bb566de 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecPool.java
@@ -17,9 +17,9 @@
*/
package org.apache.hadoop.io.compress;
-import java.util.ArrayList;
+import java.util.HashSet;
import java.util.HashMap;
-import java.util.List;
+import java.util.Set;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
@@ -47,15 +47,15 @@ public class CodecPool {
* A global compressor pool used to save the expensive
* construction/destruction of (possibly native) decompression codecs.
*/
- private static final Map<Class<Compressor>, List<Compressor>> compressorPool =
- new HashMap<Class<Compressor>, List<Compressor>>();
+ private static final Map<Class<Compressor>, Set<Compressor>> compressorPool =
+ new HashMap<Class<Compressor>, Set<Compressor>>();
/**
* A global decompressor pool used to save the expensive
* construction/destruction of (possibly native) decompression codecs.
*/
- private static final Map<Class<Decompressor>, List<Decompressor>> decompressorPool =
- new HashMap<Class<Decompressor>, List<Decompressor>>();
+ private static final Map<Class<Decompressor>, Set<Decompressor>> decompressorPool =
+ new HashMap<Class<Decompressor>, Set<Decompressor>>();
private static <T> LoadingCache<Class<T>, AtomicInteger> createCache(
Class<T> klass) {
@@ -80,20 +80,21 @@ public class CodecPool {
private static final LoadingCache<Class<Decompressor>, AtomicInteger> decompressorCounts =
createCache(Decompressor.class);
- private static <T> T borrow(Map<Class<T>, List<T>> pool,
+ private static <T> T borrow(Map<Class<T>, Set<T>> pool,
Class<? extends T> codecClass) {
T codec = null;
// Check if an appropriate codec is available
- List<T> codecList;
+ Set<T> codecSet;
synchronized (pool) {
- codecList = pool.get(codecClass);
+ codecSet = pool.get(codecClass);
}
- if (codecList != null) {
- synchronized (codecList) {
- if (!codecList.isEmpty()) {
- codec = codecList.remove(codecList.size() - 1);
+ if (codecSet != null) {
+ synchronized (codecSet) {
+ if (!codecSet.isEmpty()) {
+ codec = codecSet.iterator().next();
+ codecSet.remove(codec);
}
}
}
@@ -101,22 +102,23 @@ public class CodecPool {
return codec;
}
- private static <T> void payback(Map<Class<T>, List<T>> pool, T codec) {
+ private static <T> boolean payback(Map<Class<T>, Set<T>> pool, T codec) {
if (codec != null) {
Class<T> codecClass = ReflectionUtils.getClass(codec);
- List<T> codecList;
+ Set<T> codecSet;
synchronized (pool) {
- codecList = pool.get(codecClass);
- if (codecList == null) {
- codecList = new ArrayList<T>();
- pool.put(codecClass, codecList);
+ codecSet = pool.get(codecClass);
+ if (codecSet == null) {
+ codecSet = new HashSet<T>();
+ pool.put(codecClass, codecSet);
}
}
- synchronized (codecList) {
- codecList.add(codec);
+ synchronized (codecSet) {
+ return codecSet.add(codec);
}
}
+ return false;
}
@SuppressWarnings("unchecked")
@@ -200,8 +202,9 @@ public class CodecPool {
return;
}
compressor.reset();
- payback(compressorPool, compressor);
- updateLeaseCount(compressorCounts, compressor, -1);
+ if (payback(compressorPool, compressor)) {
+ updateLeaseCount(compressorCounts, compressor, -1);
+ }
}
/**
@@ -219,8 +222,9 @@ public class CodecPool {
return;
}
decompressor.reset();
- payback(decompressorPool, decompressor);
- updateLeaseCount(decompressorCounts, decompressor, -1);
+ if (payback(decompressorPool, decompressor)) {
+ updateLeaseCount(decompressorCounts, decompressor, -1);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a1dcce8/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
index 5cacebf..c889a59 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecPool.java
@@ -19,15 +19,9 @@ package org.apache.hadoop.io.compress;
import static org.junit.Assert.assertEquals;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
@@ -35,6 +29,9 @@ import org.apache.hadoop.conf.Configuration;
import org.junit.Before;
import org.junit.Test;
+import java.util.HashSet;
+import java.util.Set;
+
public class TestCodecPool {
private final String LEASE_COUNT_ERR =
"Incorrect number of leased (de)compressors";
@@ -61,6 +58,25 @@ public class TestCodecPool {
CodecPool.returnCompressor(comp1);
assertEquals(LEASE_COUNT_ERR, 0,
CodecPool.getLeasedCompressorsCount(codec));
+
+ CodecPool.returnCompressor(comp1);
+ assertEquals(LEASE_COUNT_ERR, 0,
+ CodecPool.getLeasedCompressorsCount(codec));
+ }
+
+ @Test(timeout = 1000)
+ public void testCompressorNotReturnSameInstance() {
+ Compressor comp = CodecPool.getCompressor(codec);
+ CodecPool.returnCompressor(comp);
+ CodecPool.returnCompressor(comp);
+ Set<Compressor> compressors = new HashSet<Compressor>();
+ for (int i = 0; i < 10; ++i) {
+ compressors.add(CodecPool.getCompressor(codec));
+ }
+ assertEquals(10, compressors.size());
+ for (Compressor compressor : compressors) {
+ CodecPool.returnCompressor(compressor);
+ }
}
@Test(timeout = 1000)
@@ -78,6 +94,10 @@ public class TestCodecPool {
CodecPool.returnDecompressor(decomp1);
assertEquals(LEASE_COUNT_ERR, 0,
CodecPool.getLeasedDecompressorsCount(codec));
+
+ CodecPool.returnDecompressor(decomp1);
+ assertEquals(LEASE_COUNT_ERR, 0,
+ CodecPool.getLeasedCompressorsCount(codec));
}
@Test(timeout = 1000)
@@ -154,4 +174,19 @@ public class TestCodecPool {
assertEquals(LEASE_COUNT_ERR, 0,
CodecPool.getLeasedDecompressorsCount(codec));
}
+
+ @Test(timeout = 1000)
+ public void testDecompressorNotReturnSameInstance() {
+ Decompressor decomp = CodecPool.getDecompressor(codec);
+ CodecPool.returnDecompressor(decomp);
+ CodecPool.returnDecompressor(decomp);
+ Set<Decompressor> decompressors = new HashSet<Decompressor>();
+ for (int i = 0; i < 10; ++i) {
+ decompressors.add(CodecPool.getDecompressor(codec));
+ }
+ assertEquals(10, decompressors.size());
+ for (Decompressor decompressor : decompressors) {
+ CodecPool.returnDecompressor(decompressor);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a1dcce8/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a8ca13f..28595ca 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -236,6 +236,9 @@ Release 2.7.0 - UNRELEASED
BUG FIXES
+ MAPREDUCE-5918. LineRecordReader can return the same decompressor to CodecPool
+ multiple times (Sergey Murylev via raviprak)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a1dcce8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java
index 6b5c26e..ba075e5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/LineRecordReader.java
@@ -284,6 +284,7 @@ public class LineRecordReader implements RecordReader<LongWritable, Text> {
} finally {
if (decompressor != null) {
CodecPool.returnDecompressor(decompressor);
+ decompressor = null;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a1dcce8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
index 880a1a2..42e94ad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/LineRecordReader.java
@@ -232,6 +232,7 @@ public class LineRecordReader extends RecordReader<LongWritable, Text> {
} finally {
if (decompressor != null) {
CodecPool.returnDecompressor(decompressor);
+ decompressor = null;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a1dcce8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
index 7b664e9..a7a87c9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestLineRecordReader.java
@@ -27,12 +27,17 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.CodecPool;
+import org.apache.hadoop.io.compress.Decompressor;
import org.junit.Test;
public class TestLineRecordReader {
@@ -225,4 +230,36 @@ public class TestLineRecordReader {
assertTrue("BOM is not skipped", skipBOM);
}
+
+ @Test
+ public void testMultipleClose() throws IOException {
+ URL testFileUrl = getClass().getClassLoader().
+ getResource("recordSpanningMultipleSplits.txt.bz2");
+ assertNotNull("Cannot find recordSpanningMultipleSplits.txt.bz2",
+ testFileUrl);
+ File testFile = new File(testFileUrl.getFile());
+ Path testFilePath = new Path(testFile.getAbsolutePath());
+ long testFileSize = testFile.length();
+ Configuration conf = new Configuration();
+ conf.setInt(org.apache.hadoop.mapreduce.lib.input.
+ LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);
+ FileSplit split = new FileSplit(testFilePath, 0, testFileSize,
+ (String[])null);
+
+ LineRecordReader reader = new LineRecordReader(conf, split);
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+ //noinspection StatementWithEmptyBody
+ while (reader.next(key, value)) ;
+ reader.close();
+ reader.close();
+
+ BZip2Codec codec = new BZip2Codec();
+ codec.setConf(conf);
+ Set<Decompressor> decompressors = new HashSet<Decompressor>();
+ for (int i = 0; i < 10; ++i) {
+ decompressors.add(CodecPool.getDecompressor(codec));
+ }
+ assertEquals(10, decompressors.size());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a1dcce8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
index a1b5147..52fdc09 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestLineRecordReader.java
@@ -27,10 +27,15 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.CodecPool;
+import org.apache.hadoop.io.compress.Decompressor;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
@@ -231,4 +236,37 @@ public class TestLineRecordReader {
assertTrue("BOM is not skipped", skipBOM);
}
+
+ @Test
+ public void testMultipleClose() throws IOException {
+ URL testFileUrl = getClass().getClassLoader().
+ getResource("recordSpanningMultipleSplits.txt.bz2");
+ assertNotNull("Cannot find recordSpanningMultipleSplits.txt.bz2",
+ testFileUrl);
+ File testFile = new File(testFileUrl.getFile());
+ Path testFilePath = new Path(testFile.getAbsolutePath());
+ long testFileSize = testFile.length();
+ Configuration conf = new Configuration();
+ conf.setInt(org.apache.hadoop.mapreduce.lib.input.
+ LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);
+ TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
+
+ // read the data and check whether BOM is skipped
+ FileSplit split = new FileSplit(testFilePath, 0, testFileSize, null);
+ LineRecordReader reader = new LineRecordReader();
+ reader.initialize(split, context);
+
+ //noinspection StatementWithEmptyBody
+ while (reader.nextKeyValue()) ;
+ reader.close();
+ reader.close();
+
+ BZip2Codec codec = new BZip2Codec();
+ codec.setConf(conf);
+ Set<Decompressor> decompressors = new HashSet<Decompressor>();
+ for (int i = 0; i < 10; ++i) {
+ decompressors.add(CodecPool.getDecompressor(codec));
+ }
+ assertEquals(10, decompressors.size());
+ }
}