You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2016/08/23 22:31:13 UTC
[01/50] [abbrv] hadoop git commit: YARN=5526.
DrainDispacher#serviceStop blocked if setDrainEventsOnStop invoked (sandflee
via Varun Saxena)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-9806 c2bcffb34 -> c37346d0e
YARN=5526. DrainDispacher#serviceStop blocked if setDrainEventsOnStop invoked (sandflee via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/913a8951
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/913a8951
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/913a8951
Branch: refs/heads/HDFS-9806
Commit: 913a8951366a5d816bbae253b230250e0b74849e
Parents: 20f0eb8
Author: Varun Saxena <va...@apache.org>
Authored: Thu Aug 18 11:38:13 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Thu Aug 18 11:38:13 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/event/AsyncDispatcher.java | 6 +++++-
.../java/org/apache/hadoop/yarn/event/DrainDispatcher.java | 5 +++++
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913a8951/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 5dea1c8..89b5861 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -148,7 +148,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
YarnConfiguration.DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT);
synchronized (waitForDrained) {
- while (!drained && eventHandlingThread != null
+ while (!isDrained() && eventHandlingThread != null
&& eventHandlingThread.isAlive()
&& System.currentTimeMillis() < endTime) {
waitForDrained.wait(1000);
@@ -303,4 +303,8 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
protected boolean isEventThreadWaiting() {
return eventHandlingThread.getState() == Thread.State.WAITING;
}
+
+ protected boolean isDrained() {
+ return drained;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913a8951/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
index cf4b1b5..f769492 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
@@ -95,6 +95,11 @@ public class DrainDispatcher extends AsyncDispatcher {
}
@Override
+ protected boolean isDrained() {
+ return drained;
+ }
+
+ @Override
protected void serviceStop() throws Exception {
stopped = true;
super.serviceStop();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: MAPREDUCE-6310. Add jdiff support
to MapReduce. (Li Lu/vinodkv via wangda)
Posted by cd...@apache.org.
MAPREDUCE-6310. Add jdiff support to MapReduce. (Li Lu/vinodkv via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d937457
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d937457
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d937457
Branch: refs/heads/HDFS-9806
Commit: 3d937457ee60d617d71b36bb6a29344e771670b4
Parents: 723facf
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Aug 19 16:26:29 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Aug 19 16:26:29 2016 -0700
----------------------------------------------------------------------
.../Apache_Hadoop_MapReduce_Common_2.6.0.xml | 281 +
.../Apache_Hadoop_MapReduce_Common_2.7.2.xml | 836 +
.../Apache_Hadoop_MapReduce_Core_2.6.0.xml | 30767 ++++++++++++++++
.../Apache_Hadoop_MapReduce_Core_2.7.2.xml | 31129 +++++++++++++++++
.../Apache_Hadoop_MapReduce_JobClient_2.6.0.xml | 975 +
.../Apache_Hadoop_MapReduce_JobClient_2.7.2.xml | 990 +
.../dev-support/jdiff/Null.java | 20 +
.../hadoop-mapreduce-client-common/pom.xml | 2 +
.../hadoop-mapreduce-client-core/pom.xml | 2 +
.../hadoop-mapreduce-client-jobclient/pom.xml | 2 +
.../hadoop-mapreduce-client/pom.xml | 127 +
11 files changed, 65131 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.6.0.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.6.0.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.6.0.xml
new file mode 100644
index 0000000..66e206b
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.6.0.xml
@@ -0,0 +1,281 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon Jun 13 17:08:29 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="hadoop-mapreduce-client-common 2.6.0"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-annotations.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/jdiff.jar -verbose -classpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/classes:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-2.6.0.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-2.6.0.jar:/Users/vinodkv/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2
.jar:/Users/vinodkv/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/vinodkv/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/vinodkv/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/vinodkv/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/vinodkv/.m2/repository/
org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/vinodkv/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/vinodkv/.m2/repository/com/google/inject/guice/3.0/guice-3.0.jar:/Users/vinodkv/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/Users/vinodkv/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/vinodkv/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/vinodkv/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/Users/vinodkv/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop
-yarn/hadoop-yarn-client/target/hadoop-yarn-client-2.6.0.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-2.6.0.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/hadoop-yarn-server-common-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/vinodkv/.m2/repository/jline/jline/0.9.94/jline-0.9.94.jar:/Users/vinodkv/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/Users/vinodkv/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/vinodkv/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/vinodkv/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/vinodkv/.m2/repository/org/xerial/snappy/snappy-java/1.0.
4.1/snappy-java-1.0.4.1.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-common/target/hadoop-common-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/vinodkv/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/vinodkv/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/vinodkv/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/vinodkv/.m2/repository/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/Users/vinodkv/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/vinodkv/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0
.4.jar:/Users/vinodkv/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/vinodkv/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/vinodkv/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org
/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-framework/2.6.0/curator-framework-2.6.0.jar:/Users/vinodkv/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-client/2.6.0/curator-client-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-recipes/2.6.0/curator-recipes-2.6.0.jar:/Users/vinodkv/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/Users/vinodkv/.m2/repository/org/htrace/htrace-core/3.0.4/htrace-core-3.0.4.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-log4j12/1.7.5/slf4j-log4j12-1.7.5.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.6.0.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home/lib/tools.jar:/Users/vinodkv/.m2/repository
/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar:/Users/vinodkv/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/vinodkv/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/vinodkv/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/vinodkv/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/vinodkv/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/vinodkv/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/vinodkv/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar -sourcepath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java -apidir /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/site/
jdiff/xml -apiname hadoop-mapreduce-client-common 2.6.0 -->
+<package name="org.apache.hadoop.mapred">
+</package>
+<package name="org.apache.hadoop.mapreduce">
+ <!-- start class org.apache.hadoop.mapreduce.TypeConverter -->
+ <class name="TypeConverter" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="TypeConverter"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.JobID"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.v2.api.records.JobId"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.JobID"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appID" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.JobId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.JobID"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.TaskType"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskType" type="org.apache.hadoop.mapreduce.v2.api.records.TaskType"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskType"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskType" type="org.apache.hadoop.mapreduce.TaskType"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskID"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.v2.api.records.TaskId"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.TaskID"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="state" type="org.apache.hadoop.mapred.TaskStatus.State"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.Phase"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="phase" type="org.apache.hadoop.mapred.TaskStatus.Phase"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="newEvents" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[]"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskCompletionEvent"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="newEvent" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="newStatus" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskAttemptID"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="yCntrs" type="org.apache.hadoop.mapreduce.v2.api.records.Counters"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counters" type="org.apache.hadoop.mapred.Counters"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counters" type="org.apache.hadoop.mapreduce.Counters"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobreport" type="org.apache.hadoop.mapreduce.v2.api.records.JobReport"/>
+ <param name="trackingUrl" type="java.lang.String"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.QueueState"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="state" type="org.apache.hadoop.yarn.api.records.QueueState"/>
+ </method>
+ <method name="fromYarn" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="state" type="org.apache.hadoop.mapreduce.v2.api.records.JobState"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TIPStatus"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="state" type="org.apache.hadoop.mapreduce.v2.api.records.TaskState"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.TaskReport"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="report" type="org.apache.hadoop.mapreduce.v2.api.records.TaskReport"/>
+ </method>
+ <method name="fromYarn" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskReports" type="java.util.List"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.JobStatus.State"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="yarnApplicationState" type="org.apache.hadoop.yarn.api.records.YarnApplicationState"/>
+ <param name="finalApplicationStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.TaskTrackerInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="node" type="org.apache.hadoop.yarn.api.records.NodeReport"/>
+ </method>
+ <method name="fromYarnNodes" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="nodes" type="java.util.List"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="application" type="org.apache.hadoop.yarn.api.records.ApplicationReport"/>
+ <param name="jobFile" type="java.lang.String"/>
+ </method>
+ <method name="fromYarnApps" return="org.apache.hadoop.mapreduce.JobStatus[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applications" type="java.util.List"/>
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.QueueInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queueInfo" type="org.apache.hadoop.yarn.api.records.QueueInfo"/>
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="fromYarnQueueInfo" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queues" type="java.util.List"/>
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="fromYarnQueueUserAclsInfo" return="org.apache.hadoop.mapreduce.QueueAclsInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="userAcls" type="java.util.List"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapreduce.TypeConverter -->
+</package>
+<package name="org.apache.hadoop.mapreduce.v2.security">
+</package>
+<package name="org.apache.hadoop.yarn.proto">
+ <!-- start interface org.apache.hadoop.yarn.proto.HSClientProtocol -->
+ <interface name="HSClientProtocol" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Fake protocol to differentiate the blocking interfaces in the
+ security info class loaders.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.yarn.proto.HSClientProtocol -->
+ <!-- start class org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService -->
+ <class name="HSClientProtocol.HSClientProtocolService" extends="java.lang.Object"
+ abstract="true"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HSClientProtocol.HSClientProtocolService"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="newReflectiveBlockingService" return="com.google.protobuf.BlockingService"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="impl" type="org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService.BlockingInterface"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService -->
+ <!-- start interface org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService.BlockingInterface -->
+ <interface name="HSClientProtocol.HSClientProtocolService.BlockingInterface" abstract="true"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB"/>
+ </interface>
+ <!-- end interface org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService.BlockingInterface -->
+</package>
+
+</api>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.7.2.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.7.2.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.7.2.xml
new file mode 100644
index 0000000..c20349b
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.7.2.xml
@@ -0,0 +1,836 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon Jun 13 20:33:05 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="hadoop-mapreduce-client-common 2.7.2"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-annotations.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/jdiff.jar -verbose -classpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/classes:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-2.7.2.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-2.7.2.jar:/Users/vinodkv/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2
.jar:/Users/vinodkv/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/vinodkv/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/vinodkv/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/vinodkv/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/vinodkv/.m2/repository/
org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/vinodkv/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/vinodkv/.m2/repository/com/google/inject/guice/3.0/guice-3.0.jar:/Users/vinodkv/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/Users/vinodkv/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/vinodkv/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/vinodkv/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/Users/vinodkv/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop
-yarn/hadoop-yarn-client/target/hadoop-yarn-client-2.7.2.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-2.7.2.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/target/hadoop-yarn-server-common-2.7.2.jar:/Users/vinodkv/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/vinodkv/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/Users/vinodkv/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/vinodkv/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/vinodkv/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/vinodkv/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/vinodkv/Workspace/eclipse-works
pace/apache-git/hadoop/hadoop-common-project/hadoop-common/target/hadoop-common-2.7.2.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/vinodkv/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/vinodkv/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/vinodkv/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/vinodkv/.m2/repository/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/Users/vinodkv/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/vinodkv/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/vinodkv/.m2/repository/commons-configuration/commons
-configuration/1.6/commons-configuration-1.6.jar:/Users/vinodkv/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/vinodkv/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.7.2.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/U
sers/vinodkv/.m2/repository/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar:/Users/vinodkv/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-recipes/2.7.1/curator-recipes-2.7.1.jar:/Users/vinodkv/.m2/repository/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar:/Users/vinodkv/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.7.2.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home/lib/tools.jar:/Users/vinodkv/.m2/repository/com/google/inject/extensions/gui
ce-servlet/3.0/guice-servlet-3.0.jar:/Users/vinodkv/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/vinodkv/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/vinodkv/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/vinodkv/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/vinodkv/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/vinodkv/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/vinodkv/.m2/repository/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar -sourcepath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java -apidir /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/site/jdiff/xml -apiname hadoop-mapredu
ce-client-common 2.7.2 -->
+<package name="org.apache.hadoop.mapred">
+</package>
+<package name="org.apache.hadoop.mapreduce">
+ <!-- start class org.apache.hadoop.mapreduce.TypeConverter -->
+ <class name="TypeConverter" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="TypeConverter"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.JobID"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.v2.api.records.JobId"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.JobID"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appID" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.JobId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.JobID"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.TaskType"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskType" type="org.apache.hadoop.mapreduce.v2.api.records.TaskType"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskType"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskType" type="org.apache.hadoop.mapreduce.TaskType"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskID"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.v2.api.records.TaskId"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.TaskID"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="state" type="org.apache.hadoop.mapred.TaskStatus.State"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.Phase"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="phase" type="org.apache.hadoop.mapred.TaskStatus.Phase"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="newEvents" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[]"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskCompletionEvent"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="newEvent" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskCompletionEvent.Status"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="newStatus" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TaskAttemptID"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapred.TaskAttemptID"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="yCntrs" type="org.apache.hadoop.mapreduce.v2.api.records.Counters"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counters" type="org.apache.hadoop.mapred.Counters"/>
+ </method>
+ <method name="toYarn" return="org.apache.hadoop.mapreduce.v2.api.records.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counters" type="org.apache.hadoop.mapreduce.Counters"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobreport" type="org.apache.hadoop.mapreduce.v2.api.records.JobReport"/>
+ <param name="trackingUrl" type="java.lang.String"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.QueueState"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="state" type="org.apache.hadoop.yarn.api.records.QueueState"/>
+ </method>
+ <method name="fromYarn" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="state" type="org.apache.hadoop.mapreduce.v2.api.records.JobState"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapred.TIPStatus"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="state" type="org.apache.hadoop.mapreduce.v2.api.records.TaskState"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.TaskReport"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="report" type="org.apache.hadoop.mapreduce.v2.api.records.TaskReport"/>
+ </method>
+ <method name="fromYarn" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskReports" type="java.util.List"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.JobStatus.State"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="yarnApplicationState" type="org.apache.hadoop.yarn.api.records.YarnApplicationState"/>
+ <param name="finalApplicationStatus" type="org.apache.hadoop.yarn.api.records.FinalApplicationStatus"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.TaskTrackerInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="node" type="org.apache.hadoop.yarn.api.records.NodeReport"/>
+ </method>
+ <method name="fromYarnNodes" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="nodes" type="java.util.List"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="application" type="org.apache.hadoop.yarn.api.records.ApplicationReport"/>
+ <param name="jobFile" type="java.lang.String"/>
+ </method>
+ <method name="fromYarnApps" return="org.apache.hadoop.mapreduce.JobStatus[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applications" type="java.util.List"/>
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="fromYarn" return="org.apache.hadoop.mapreduce.QueueInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queueInfo" type="org.apache.hadoop.yarn.api.records.QueueInfo"/>
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="fromYarnQueueInfo" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queues" type="java.util.List"/>
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="fromYarnQueueUserAclsInfo" return="org.apache.hadoop.mapreduce.QueueAclsInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="userAcls" type="java.util.List"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapreduce.TypeConverter -->
+</package>
+<package name="org.apache.hadoop.mapreduce.v2.api.protocolrecords">
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest -->
+ <interface name="CancelDelegationTokenRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setDelegationToken"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="dToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+ </method>
+ <doc>
+ <![CDATA[The request issued by the client to the {@code ResourceManager} to cancel a
+ delegation token.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse -->
+ <interface name="CancelDelegationTokenResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[The response from the {@code ResourceManager} to a cancelDelegationToken
+ request.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest -->
+ <interface name="FailTaskAttemptRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getTaskAttemptId" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setTaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskAttemptId" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse -->
+ <interface name="FailTaskAttemptResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest -->
+ <interface name="GetCountersRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getJobId" return="org.apache.hadoop.mapreduce.v2.api.records.JobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setJobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobId" type="org.apache.hadoop.mapreduce.v2.api.records.JobId"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersResponse -->
+ <interface name="GetCountersResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getCounters" return="org.apache.hadoop.mapreduce.v2.api.records.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setCounters"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counters" type="org.apache.hadoop.mapreduce.v2.api.records.Counters"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest -->
+ <interface name="GetDelegationTokenRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getRenewer" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setRenewer"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="renewer" type="java.lang.String"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse -->
+ <interface name="GetDelegationTokenResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="setDelegationToken"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="clientDToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+ </method>
+ <method name="getDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest -->
+ <interface name="GetDiagnosticsRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getTaskAttemptId" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setTaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskAttemptId" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse -->
+ <interface name="GetDiagnosticsResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getDiagnosticsList" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDiagnostics" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="index" type="int"/>
+ </method>
+ <method name="getDiagnosticsCount" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="addAllDiagnostics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="diagnostics" type="java.util.List"/>
+ </method>
+ <method name="addDiagnostics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="diagnostic" type="java.lang.String"/>
+ </method>
+ <method name="removeDiagnostics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="index" type="int"/>
+ </method>
+ <method name="clearDiagnostics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest -->
+ <interface name="GetJobReportRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getJobId" return="org.apache.hadoop.mapreduce.v2.api.records.JobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setJobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobId" type="org.apache.hadoop.mapreduce.v2.api.records.JobId"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse -->
+ <interface name="GetJobReportResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getJobReport" return="org.apache.hadoop.mapreduce.v2.api.records.JobReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setJobReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobReport" type="org.apache.hadoop.mapreduce.v2.api.records.JobReport"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest -->
+ <interface name="GetTaskAttemptCompletionEventsRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getJobId" return="org.apache.hadoop.mapreduce.v2.api.records.JobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getFromEventId" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getMaxEvents" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setJobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobId" type="org.apache.hadoop.mapreduce.v2.api.records.JobId"/>
+ </method>
+ <method name="setFromEventId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="int"/>
+ </method>
+ <method name="setMaxEvents"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="maxEvents" type="int"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsResponse -->
+ <interface name="GetTaskAttemptCompletionEventsResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getCompletionEventList" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getCompletionEvent" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="index" type="int"/>
+ </method>
+ <method name="getCompletionEventCount" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="addAllCompletionEvents"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="eventList" type="java.util.List"/>
+ </method>
+ <method name="addCompletionEvent"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="event" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent"/>
+ </method>
+ <method name="removeCompletionEvent"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="index" type="int"/>
+ </method>
+ <method name="clearCompletionEvents"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportRequest -->
+ <interface name="GetTaskAttemptReportRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getTaskAttemptId" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setTaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskAttemptId" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportResponse -->
+ <interface name="GetTaskAttemptReportResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getTaskAttemptReport" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setTaskAttemptReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskAttemptReport" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportRequest -->
+ <interface name="GetTaskReportRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getTaskId" return="org.apache.hadoop.mapreduce.v2.api.records.TaskId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setTaskId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskId" type="org.apache.hadoop.mapreduce.v2.api.records.TaskId"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportResponse -->
+ <interface name="GetTaskReportResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getTaskReport" return="org.apache.hadoop.mapreduce.v2.api.records.TaskReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setTaskReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskReport" type="org.apache.hadoop.mapreduce.v2.api.records.TaskReport"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest -->
+ <interface name="GetTaskReportsRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getJobId" return="org.apache.hadoop.mapreduce.v2.api.records.JobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getTaskType" return="org.apache.hadoop.mapreduce.v2.api.records.TaskType"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setJobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobId" type="org.apache.hadoop.mapreduce.v2.api.records.JobId"/>
+ </method>
+ <method name="setTaskType"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskType" type="org.apache.hadoop.mapreduce.v2.api.records.TaskType"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsResponse -->
+ <interface name="GetTaskReportsResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getTaskReportList" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getTaskReport" return="org.apache.hadoop.mapreduce.v2.api.records.TaskReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="index" type="int"/>
+ </method>
+ <method name="getTaskReportCount" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="addAllTaskReports"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskReports" type="java.util.List"/>
+ </method>
+ <method name="addTaskReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskReport" type="org.apache.hadoop.mapreduce.v2.api.records.TaskReport"/>
+ </method>
+ <method name="removeTaskReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="index" type="int"/>
+ </method>
+ <method name="clearTaskReports"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest -->
+ <interface name="KillJobRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getJobId" return="org.apache.hadoop.mapreduce.v2.api.records.JobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setJobId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobId" type="org.apache.hadoop.mapreduce.v2.api.records.JobId"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobResponse -->
+ <interface name="KillJobResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest -->
+ <interface name="KillTaskAttemptRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getTaskAttemptId" return="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setTaskAttemptId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskAttemptId" type="org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse -->
+ <interface name="KillTaskAttemptResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest -->
+ <interface name="KillTaskRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getTaskId" return="org.apache.hadoop.mapreduce.v2.api.records.TaskId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setTaskId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskId" type="org.apache.hadoop.mapreduce.v2.api.records.TaskId"/>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse -->
+ <interface name="KillTaskResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest -->
+ <interface name="RenewDelegationTokenRequest" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setDelegationToken"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="dToken" type="org.apache.hadoop.yarn.api.records.Token"/>
+ </method>
+ <doc>
+ <![CDATA[The request issued by the client to renew a delegation token from
+ the {@code ResourceManager}.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest -->
+ <!-- start interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse -->
+ <interface name="RenewDelegationTokenResponse" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getNextExpirationTime" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setNextExpirationTime"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="expTime" type="long"/>
+ </method>
+ <doc>
+ <![CDATA[The response to a renewDelegationToken call to the {@code ResourceManager}.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse -->
+</package>
+<package name="org.apache.hadoop.mapreduce.v2.security">
+</package>
+<package name="org.apache.hadoop.yarn.proto">
+ <!-- start interface org.apache.hadoop.yarn.proto.HSClientProtocol -->
+ <interface name="HSClientProtocol" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Fake protocol to differentiate the blocking interfaces in the
+ security info class loaders.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.yarn.proto.HSClientProtocol -->
+ <!-- start class org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService -->
+ <class name="HSClientProtocol.HSClientProtocolService" extends="java.lang.Object"
+ abstract="true"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HSClientProtocol.HSClientProtocolService"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="newReflectiveBlockingService" return="com.google.protobuf.BlockingService"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="impl" type="org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService.BlockingInterface"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService -->
+ <!-- start interface org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService.BlockingInterface -->
+ <interface name="HSClientProtocol.HSClientProtocolService.BlockingInterface" abstract="true"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.v2.api.MRClientProtocolPB"/>
+ </interface>
+ <!-- end interface org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService.BlockingInterface -->
+</package>
+
+</api>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-3388. Allocation in LeafQueue
could get stuck because DRF calculator isn't well supported when computing
user-limit. (Nathan Roberts via wangda)
Posted by cd...@apache.org.
YARN-3388. Allocation in LeafQueue could get stuck because DRF calculator isn't well supported when computing user-limit. (Nathan Roberts via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/444b2ea7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/444b2ea7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/444b2ea7
Branch: refs/heads/HDFS-9806
Commit: 444b2ea7afebf9f6c3d356154b71abfd0ea95b23
Parents: 3d93745
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Aug 19 16:28:32 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Aug 19 16:28:32 2016 -0700
----------------------------------------------------------------------
.../scheduler/capacity/LeafQueue.java | 206 +++++++++++++++++--
.../scheduler/capacity/TestLeafQueue.java | 198 +++++++++++++++++-
.../scheduler/capacity/TestUtils.java | 24 ++-
3 files changed, 396 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/444b2ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 636762f..1ca69be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.util.*;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -70,9 +73,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPo
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.server.utils.Lock.NoLock;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
@Private
@Unstable
@@ -111,7 +116,7 @@ public class LeafQueue extends AbstractCSQueue {
// cache last cluster resource to compute actual capacity
private Resource lastClusterResource = Resources.none();
-
+
private final QueueResourceLimitsInfo queueResourceLimitsInfo =
new QueueResourceLimitsInfo();
@@ -119,6 +124,10 @@ public class LeafQueue extends AbstractCSQueue {
private OrderingPolicy<FiCaSchedulerApp> orderingPolicy = null;
+ // Summation of consumed ratios for all users in queue
+ private float totalUserConsumedRatio = 0;
+ private UsageRatios qUsageRatios;
+
// record all ignore partition exclusivityRMContainer, this will be used to do
// preemption, key is the partition of the RMContainer allocated on
private Map<String, TreeSet<RMContainer>> ignorePartitionExclusivityRMContainers =
@@ -135,6 +144,8 @@ public class LeafQueue extends AbstractCSQueue {
// One time initialization is enough since it is static ordering policy
this.pendingOrderingPolicy = new FifoOrderingPolicyForPendingApps();
+ qUsageRatios = new UsageRatios();
+
if(LOG.isDebugEnabled()) {
LOG.debug("LeafQueue:" + " name=" + queueName
+ ", fullname=" + getQueuePath());
@@ -159,7 +170,7 @@ public class LeafQueue extends AbstractCSQueue {
setQueueResourceLimitsInfo(clusterResource);
CapacitySchedulerConfiguration conf = csContext.getConfiguration();
-
+
setOrderingPolicy(conf.<FiCaSchedulerApp>getOrderingPolicy(getQueuePath()));
userLimit = conf.getUserLimit(getQueuePath());
@@ -1149,6 +1160,9 @@ public class LeafQueue extends AbstractCSQueue {
private Resource computeUserLimit(FiCaSchedulerApp application,
Resource clusterResource, User user,
String nodePartition, SchedulingMode schedulingMode) {
+ Resource partitionResource = labelManager.getResourceByLabel(nodePartition,
+ clusterResource);
+
// What is our current capacity?
// * It is equal to the max(required, queue-capacity) if
// we're running below capacity. The 'max' ensures that jobs in queues
@@ -1157,7 +1171,7 @@ public class LeafQueue extends AbstractCSQueue {
// (usedResources + required) (which extra resources we are allocating)
Resource queueCapacity =
Resources.multiplyAndNormalizeUp(resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, clusterResource),
+ partitionResource,
queueCapacities.getAbsoluteCapacity(nodePartition),
minimumAllocation);
@@ -1169,15 +1183,30 @@ public class LeafQueue extends AbstractCSQueue {
// Allow progress for queues with miniscule capacity
queueCapacity =
Resources.max(
- resourceCalculator, clusterResource,
+ resourceCalculator, partitionResource,
queueCapacity,
required);
+
+ /* We want to base the userLimit calculation on
+ * max(queueCapacity, usedResources+required). However, we want
+ * usedResources to be based on the combined ratios of all the users in the
+ * queue so we use consumedRatio to calculate such.
+ * The calculation is dependent on how the resourceCalculator calculates the
+ * ratio between two Resources. DRF Example: If usedResources is
+ * greater than queueCapacity and users have the following [mem,cpu] usages:
+ * User1: [10%,20%] - Dominant resource is 20%
+ * User2: [30%,10%] - Dominant resource is 30%
+ * Then total consumedRatio is then 20+30=50%. Yes, this value can be
+ * larger than 100% but for the purposes of making sure all users are
+ * getting their fair share, it works.
+ */
+ Resource consumed = Resources.multiplyAndNormalizeUp(resourceCalculator,
+ partitionResource, qUsageRatios.getUsageRatio(nodePartition),
+ minimumAllocation);
Resource currentCapacity =
- Resources.lessThan(resourceCalculator, clusterResource,
- queueUsage.getUsed(nodePartition), queueCapacity) ? queueCapacity
- : Resources.add(queueUsage.getUsed(nodePartition), required);
-
+ Resources.lessThan(resourceCalculator, partitionResource, consumed,
+ queueCapacity) ? queueCapacity : Resources.add(consumed, required);
// Never allow a single user to take more than the
// queue's configured capacity * user-limit-factor.
// Also, the queue's configured capacity should be higher than
@@ -1186,9 +1215,10 @@ public class LeafQueue extends AbstractCSQueue {
final int activeUsers = activeUsersManager.getNumActiveUsers();
// User limit resource is determined by:
- // max{currentCapacity / #activeUsers, currentCapacity * user-limit-percentage%)
+ // max{currentCapacity / #activeUsers, currentCapacity *
+ // user-limit-percentage%)
Resource userLimitResource = Resources.max(
- resourceCalculator, clusterResource,
+ resourceCalculator, partitionResource,
Resources.divideAndCeil(
resourceCalculator, currentCapacity, activeUsers),
Resources.divideAndCeil(
@@ -1212,8 +1242,7 @@ public class LeafQueue extends AbstractCSQueue {
maxUserLimit =
Resources.multiplyAndRoundDown(queueCapacity, userLimitFactor);
} else if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
- maxUserLimit =
- labelManager.getResourceByLabel(nodePartition, clusterResource);
+ maxUserLimit = partitionResource;
}
// Cap final user limit with maxUserLimit
@@ -1221,7 +1250,7 @@ public class LeafQueue extends AbstractCSQueue {
Resources.roundUp(
resourceCalculator,
Resources.min(
- resourceCalculator, clusterResource,
+ resourceCalculator, partitionResource,
userLimitResource,
maxUserLimit
),
@@ -1229,18 +1258,22 @@ public class LeafQueue extends AbstractCSQueue {
if (LOG.isDebugEnabled()) {
String userName = application.getUser();
- LOG.debug("User limit computation for " + userName +
+ LOG.debug("User limit computation for " + userName +
" in queue " + getQueueName() +
" userLimitPercent=" + userLimit +
" userLimitFactor=" + userLimitFactor +
- " required: " + required +
- " consumed: " + user.getUsed() +
+ " required: " + required +
+ " consumed: " + consumed +
" user-limit-resource: " + userLimitResource +
- " queueCapacity: " + queueCapacity +
+ " queueCapacity: " + queueCapacity +
" qconsumed: " + queueUsage.getUsed() +
+ " consumedRatio: " + totalUserConsumedRatio +
" currentCapacity: " + currentCapacity +
" activeUsers: " + activeUsers +
- " clusterCapacity: " + clusterResource
+ " clusterCapacity: " + clusterResource +
+ " resourceByLabel: " + partitionResource +
+ " usageratio: " + qUsageRatios.getUsageRatio(nodePartition) +
+ " Partition: " + nodePartition
);
}
user.setUserResourceLimit(userLimitResource);
@@ -1347,6 +1380,42 @@ public class LeafQueue extends AbstractCSQueue {
}
}
+ private synchronized float calculateUserUsageRatio(Resource clusterResource,
+ String nodePartition) {
+ Resource resourceByLabel =
+ labelManager.getResourceByLabel(nodePartition, clusterResource);
+ float consumed = 0;
+ User user;
+ for (Map.Entry<String, User> entry : users.entrySet()) {
+ user = entry.getValue();
+ consumed += user.resetAndUpdateUsageRatio(resourceCalculator,
+ resourceByLabel, nodePartition);
+ }
+ return consumed;
+ }
+
+ private synchronized void recalculateQueueUsageRatio(Resource clusterResource,
+ String nodePartition) {
+ ResourceUsage queueResourceUsage = this.getQueueResourceUsage();
+
+ if (nodePartition == null) {
+ for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(),
+ queueResourceUsage.getNodePartitionsSet())) {
+ qUsageRatios.setUsageRatio(partition,
+ calculateUserUsageRatio(clusterResource, partition));
+ }
+ } else {
+ qUsageRatios.setUsageRatio(nodePartition,
+ calculateUserUsageRatio(clusterResource, nodePartition));
+ }
+ }
+
+ private synchronized void updateQueueUsageRatio(String nodePartition,
+ float delta) {
+ qUsageRatios.incUsageRatio(nodePartition, delta);
+ }
+
+
@Override
public void completedContainer(Resource clusterResource,
FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer,
@@ -1384,7 +1453,7 @@ public class LeafQueue extends AbstractCSQueue {
removed =
application.containerCompleted(rmContainer, containerStatus,
event, node.getPartition());
-
+
node.releaseContainer(container);
}
@@ -1417,6 +1486,8 @@ public class LeafQueue extends AbstractCSQueue {
boolean isIncreasedAllocation) {
super.allocateResource(clusterResource, resource, nodePartition,
isIncreasedAllocation);
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition,
+ clusterResource);
// handle ignore exclusivity container
if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
@@ -1435,6 +1506,12 @@ public class LeafQueue extends AbstractCSQueue {
String userName = application.getUser();
User user = getUser(userName);
user.assignContainer(resource, nodePartition);
+
+ // Update usage ratios
+ updateQueueUsageRatio(nodePartition,
+ user.updateUsageRatio(resourceCalculator, resourceByLabel,
+ nodePartition));
+
// Note this is a bit unconventional since it gets the object and modifies
// it here, rather then using set routine
Resources.subtractFrom(application.getHeadroom(), resource); // headroom
@@ -1455,6 +1532,8 @@ public class LeafQueue extends AbstractCSQueue {
RMContainer rmContainer, boolean isChangeResource) {
super.releaseResource(clusterResource, resource, nodePartition,
isChangeResource);
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition,
+ clusterResource);
// handle ignore exclusivity container
if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
@@ -1474,6 +1553,12 @@ public class LeafQueue extends AbstractCSQueue {
String userName = application.getUser();
User user = getUser(userName);
user.releaseContainer(resource, nodePartition);
+
+ // Update usage ratios
+ updateQueueUsageRatio(nodePartition,
+ user.updateUsageRatio(resourceCalculator, resourceByLabel,
+ nodePartition));
+
metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
if (LOG.isDebugEnabled()) {
@@ -1513,7 +1598,10 @@ public class LeafQueue extends AbstractCSQueue {
// absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity
// during allocation
setQueueResourceLimitsInfo(clusterResource);
-
+
+ // Update user consumedRatios
+ recalculateQueueUsageRatio(clusterResource, null);
+
// Update metrics
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
minimumAllocation, this, labelManager, null);
@@ -1565,17 +1653,93 @@ public class LeafQueue extends AbstractCSQueue {
queueUsage.decAMUsed(nodeLabel, resourceToDec);
}
+ /*
+ * Usage Ratio
+ */
+ static private class UsageRatios {
+ private Map<String, Float> usageRatios;
+ private ReadLock readLock;
+ private WriteLock writeLock;
+
+ public UsageRatios() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+ usageRatios = new HashMap<String, Float>();
+ }
+
+ private void incUsageRatio(String label, float delta) {
+ try {
+ writeLock.lock();
+ Float fl = usageRatios.get(label);
+ if (null == fl) {
+ fl = new Float(0.0);
+ }
+ fl += delta;
+ usageRatios.put(label, new Float(fl));
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ float getUsageRatio(String label) {
+ try {
+ readLock.lock();
+ Float f = usageRatios.get(label);
+ if (null == f) {
+ return 0.0f;
+ }
+ return f;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ private void setUsageRatio(String label, float ratio) {
+ try {
+ writeLock.lock();
+ usageRatios.put(label, new Float(ratio));
+ } finally {
+ writeLock.unlock();
+ }
+ }
+ }
+
+ @VisibleForTesting
+ public float getUsageRatio(String label) {
+ return qUsageRatios.getUsageRatio(label);
+ }
+
@VisibleForTesting
public static class User {
ResourceUsage userResourceUsage = new ResourceUsage();
volatile Resource userResourceLimit = Resource.newInstance(0, 0);
int pendingApplications = 0;
int activeApplications = 0;
+ private UsageRatios userUsageRatios = new UsageRatios();
public ResourceUsage getResourceUsage() {
return userResourceUsage;
}
+ public synchronized float resetAndUpdateUsageRatio(
+ ResourceCalculator resourceCalculator,
+ Resource resource, String nodePartition) {
+ userUsageRatios.setUsageRatio(nodePartition, 0);
+ return updateUsageRatio(resourceCalculator, resource, nodePartition);
+ }
+
+ public synchronized float updateUsageRatio(
+ ResourceCalculator resourceCalculator,
+ Resource resource, String nodePartition) {
+ float delta;
+ float newRatio =
+ Resources.ratio(resourceCalculator, getUsed(nodePartition), resource);
+ delta = newRatio - userUsageRatios.getUsageRatio(nodePartition);
+ userUsageRatios.setUsageRatio(nodePartition, newRatio);
+ return delta;
+ }
+
public Resource getUsed() {
return userResourceUsage.getUsed();
}
@@ -1713,7 +1877,7 @@ public class LeafQueue extends AbstractCSQueue {
.getSchedulableEntities()) {
apps.add(pendingApp.getApplicationAttemptId());
}
- for (FiCaSchedulerApp app :
+ for (FiCaSchedulerApp app :
orderingPolicy.getSchedulableEntities()) {
apps.add(app.getApplicationAttemptId());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/444b2ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 274c063..b2c53da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -41,6 +41,8 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CyclicBarrier;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -72,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerRequestK
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue.User;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
@@ -83,6 +86,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
@@ -97,6 +101,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.T
public class TestLeafQueue {
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
+ private static final Log LOG = LogFactory.getLog(TestLeafQueue.class);
RMContext rmContext;
RMContext spyRMContext;
@@ -106,16 +111,29 @@ public class TestLeafQueue {
CapacitySchedulerContext csContext;
CSQueue root;
- Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
+ Map<String, CSQueue> queues;
final static int GB = 1024;
final static String DEFAULT_RACK = "/default";
- private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
+ private final ResourceCalculator resourceCalculator =
+ new DefaultResourceCalculator();
+ private final ResourceCalculator dominantResourceCalculator =
+ new DominantResourceCalculator();
+
@Before
public void setUp() throws Exception {
+ setUpInternal(resourceCalculator);
+ }
+
+ private void setUpWithDominantResourceCalculator() throws Exception {
+ setUpInternal(dominantResourceCalculator);
+ }
+
+ private void setUpInternal(ResourceCalculator rC) throws Exception {
CapacityScheduler spyCs = new CapacityScheduler();
+ queues = new HashMap<String, CSQueue>();
cs = spy(spyCs);
rmContext = TestUtils.getMockRMContext();
spyRMContext = spy(rmContext);
@@ -134,6 +152,8 @@ public class TestLeafQueue {
csConf =
new CapacitySchedulerConfiguration();
csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true);
+ csConf.setBoolean(
+ "yarn.scheduler.capacity.reservations-continue-look-all-nodes", false);
final String newRoot = "root" + System.currentTimeMillis();
setupQueueConfiguration(csConf, newRoot);
YarnConfiguration conf = new YarnConfiguration();
@@ -153,6 +173,7 @@ public class TestLeafQueue {
when(csContext.getResourceCalculator()).
thenReturn(resourceCalculator);
when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
+ when(csContext.getResourceCalculator()).thenReturn(rC);
when(csContext.getRMContext()).thenReturn(rmContext);
RMContainerTokenSecretManager containerTokenSecretManager =
new RMContainerTokenSecretManager(conf);
@@ -179,7 +200,8 @@ public class TestLeafQueue {
.thenReturn(new YarnConfiguration());
when(cs.getNumClusterNodes()).thenReturn(3);
}
-
+
+
private static final String A = "a";
private static final String B = "b";
private static final String C = "c";
@@ -608,14 +630,180 @@ public class TestLeafQueue {
assertEquals((int)(a.getCapacity() * node_0.getTotalResource().getMemorySize()),
a.getMetrics().getAvailableMB());
}
-
+ @Test
+ public void testDRFUsageRatioRounding() throws Exception {
+ CSAssignment assign;
+ setUpWithDominantResourceCalculator();
+ // Mock the queue
+ LeafQueue b = stubLeafQueue((LeafQueue) queues.get(E));
+
+ // Users
+ final String user0 = "user_0";
+
+ // Submit applications
+ final ApplicationAttemptId appAttemptId0 =
+ TestUtils.getMockApplicationAttemptId(0, 0);
+ FiCaSchedulerApp app0 =
+ new FiCaSchedulerApp(appAttemptId0, user0, b,
+ b.getActiveUsersManager(), spyRMContext);
+ b.submitApplicationAttempt(app0, user0);
+
+ // Setup some nodes
+ String host0 = "127.0.0.1";
+ FiCaSchedulerNode node0 =
+ TestUtils.getMockNode(host0, DEFAULT_RACK, 0, 80 * GB, 100);
+
+ // Make cluster relatively large so usageRatios are small
+ int numNodes = 1000;
+ Resource clusterResource =
+ Resources.createResource(numNodes * (80 * GB), numNodes * 100);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+
+ // Set user-limit. Need a small queue within a large cluster.
+ b.setUserLimit(50);
+ b.setUserLimitFactor(1000000);
+ b.setMaxCapacity(1.0f);
+ b.setAbsoluteCapacity(0.00001f);
+
+ // First allocation is larger than second but is still vcore dominant
+ // so usage ratio will be based on vcores. If consumedRatio doesn't round
+ // in our favor then new limit calculation will actually be less than
+ // what is currently consumed and we will fail to allocate
+ Priority priority = TestUtils.createMockPriority(1);
+ app0.updateResourceRequests(Collections.singletonList(TestUtils
+ .createResourceRequest(ResourceRequest.ANY, 20 * GB, 29, 1, true,
+ priority, recordFactory, RMNodeLabelsManager.NO_LABEL)));
+ assign = b.assignContainers(clusterResource, node0, new ResourceLimits(
+ clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
+ app0.updateResourceRequests(Collections.singletonList(TestUtils
+ .createResourceRequest(ResourceRequest.ANY, 10 * GB, 29, 2, true,
+ priority, recordFactory, RMNodeLabelsManager.NO_LABEL)));
+ assign = b.assignContainers(clusterResource, node0, new ResourceLimits(
+ clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
+ assertTrue("Still within limits, should assign",
+ assign.getResource().getMemorySize() > 0);
+ }
+
+ @Test
+ public void testDRFUserLimits() throws Exception {
+ setUpWithDominantResourceCalculator();
+
+ // Mock the queue
+ LeafQueue b = stubLeafQueue((LeafQueue) queues.get(B));
+ // unset maxCapacity
+ b.setMaxCapacity(1.0f);
+
+ // Users
+ final String user0 = "user_0";
+ final String user1 = "user_1";
+
+ // Submit applications
+ final ApplicationAttemptId appAttemptId0 =
+ TestUtils.getMockApplicationAttemptId(0, 0);
+ FiCaSchedulerApp app0 =
+ new FiCaSchedulerApp(appAttemptId0, user0, b,
+ b.getActiveUsersManager(), spyRMContext);
+ b.submitApplicationAttempt(app0, user0);
+
+ final ApplicationAttemptId appAttemptId2 =
+ TestUtils.getMockApplicationAttemptId(2, 0);
+ FiCaSchedulerApp app2 =
+ new FiCaSchedulerApp(appAttemptId2, user1, b,
+ b.getActiveUsersManager(), spyRMContext);
+ b.submitApplicationAttempt(app2, user1);
+
+ // Setup some nodes
+ String host0 = "127.0.0.1";
+ FiCaSchedulerNode node0 =
+ TestUtils.getMockNode(host0, DEFAULT_RACK, 0, 8 * GB, 100);
+ String host1 = "127.0.0.2";
+ FiCaSchedulerNode node1 =
+ TestUtils.getMockNode(host1, DEFAULT_RACK, 0, 8 * GB, 100);
+
+ int numNodes = 2;
+ Resource clusterResource =
+ Resources.createResource(numNodes * (8 * GB), numNodes * 100);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+
+ // Setup resource-requests so that one application is memory dominant
+ // and other application is vcores dominant
+ Priority priority = TestUtils.createMockPriority(1);
+ app0.updateResourceRequests(Collections.singletonList(TestUtils
+ .createResourceRequest(ResourceRequest.ANY, 1 * GB, 40, 10, true,
+ priority, recordFactory, RMNodeLabelsManager.NO_LABEL)));
+
+ app2.updateResourceRequests(Collections.singletonList(TestUtils
+ .createResourceRequest(ResourceRequest.ANY, 2 * GB, 10, 10, true,
+ priority, recordFactory, RMNodeLabelsManager.NO_LABEL)));
+
+ /**
+ * Start testing...
+ */
+
+ // Set user-limit
+ b.setUserLimit(50);
+ b.setUserLimitFactor(2);
+ User queueUser0 = b.getUser(user0);
+ User queueUser1 = b.getUser(user1);
+
+ assertEquals("There should 2 active users!", 2, b
+ .getActiveUsersManager().getNumActiveUsers());
+ // Fill both Nodes as far as we can
+ CSAssignment assign;
+ do {
+ assign =
+ b.assignContainers(clusterResource, node0, new ResourceLimits(
+ clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
+ LOG.info(assign.toString());
+ } while (assign.getResource().getMemorySize() > 0 &&
+ assign.getAssignmentInformation().getNumReservations() == 0);
+ do {
+ assign =
+ b.assignContainers(clusterResource, node1, new ResourceLimits(
+ clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
+ } while (assign.getResource().getMemorySize() > 0 &&
+ assign.getAssignmentInformation().getNumReservations() == 0);
+ //LOG.info("user_0: " + queueUser0.getUsed());
+ //LOG.info("user_1: " + queueUser1.getUsed());
+
+ assertTrue("Verify user_0 got resources ", queueUser0.getUsed()
+ .getMemorySize() > 0);
+ assertTrue("Verify user_1 got resources ", queueUser1.getUsed()
+ .getMemorySize() > 0);
+ assertTrue(
+ "Exepected AbsoluteUsedCapacity > 0.95, got: "
+ + b.getAbsoluteUsedCapacity(), b.getAbsoluteUsedCapacity() > 0.95);
+
+ // Verify consumedRatio is based on dominant resources
+ float expectedRatio =
+ queueUser0.getUsed().getVirtualCores()
+ / (numNodes * 100.0f)
+ + queueUser1.getUsed().getMemorySize()
+ / (numNodes * 8.0f * GB);
+ assertEquals(expectedRatio, b.getUsageRatio(""), 0.001);
+ // Add another node and make sure consumedRatio is adjusted
+ // accordingly.
+ numNodes = 3;
+ clusterResource =
+ Resources.createResource(numNodes * (8 * GB), numNodes * 100);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
+ expectedRatio =
+ queueUser0.getUsed().getVirtualCores()
+ / (numNodes * 100.0f)
+ + queueUser1.getUsed().getMemorySize()
+ / (numNodes * 8.0f * GB);
+ assertEquals(expectedRatio, b.getUsageRatio(""), 0.001);
+ }
+
@Test
public void testUserLimits() throws Exception {
// Mock the queue
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
//unset maxCapacity
a.setMaxCapacity(1.0f);
-
+
// Users
final String user_0 = "user_0";
final String user_1 = "user_1";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/444b2ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 66e833f..a60b7ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -161,10 +161,17 @@ public class TestUtils {
public static ResourceRequest createResourceRequest(
String resourceName, int memory, int numContainers, boolean relaxLocality,
Priority priority, RecordFactory recordFactory, String labelExpression) {
- ResourceRequest request =
+ return createResourceRequest(resourceName, memory, 1, numContainers,
+ relaxLocality, priority, recordFactory, labelExpression);
+ }
+
+ public static ResourceRequest createResourceRequest(String resourceName,
+ int memory, int vcores, int numContainers, boolean relaxLocality,
+ Priority priority, RecordFactory recordFactory, String labelExpression) {
+ ResourceRequest request =
recordFactory.newRecordInstance(ResourceRequest.class);
- Resource capability = Resources.createResource(memory, 1);
-
+ Resource capability = Resources.createResource(memory, vcores);
+
request.setNumContainers(numContainers);
request.setResourceName(resourceName);
request.setCapability(capability);
@@ -192,13 +199,18 @@ public class TestUtils {
return ApplicationAttemptId.newInstance(applicationId, attemptId);
}
- public static FiCaSchedulerNode getMockNode(
- String host, String rack, int port, int capability) {
+ public static FiCaSchedulerNode getMockNode(String host, String rack,
+ int port, int memory) {
+ return getMockNode(host, rack, port, memory, 1);
+ }
+
+ public static FiCaSchedulerNode getMockNode(String host, String rack,
+ int port, int memory, int vcores) {
NodeId nodeId = NodeId.newInstance(host, port);
RMNode rmNode = mock(RMNode.class);
when(rmNode.getNodeID()).thenReturn(nodeId);
when(rmNode.getTotalCapability()).thenReturn(
- Resources.createResource(capability, 1));
+ Resources.createResource(memory, vcores));
when(rmNode.getNodeAddress()).thenReturn(host+":"+port);
when(rmNode.getHostName()).thenReturn(host);
when(rmNode.getRackName()).thenReturn(rack);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HADOOP-13503. Improve
SaslRpcClient failure logging. Contributed by Xiaobing Zhou.
Posted by cd...@apache.org.
HADOOP-13503. Improve SaslRpcClient failure logging. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c5c3e81b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c5c3e81b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c5c3e81b
Branch: refs/heads/HDFS-9806
Commit: c5c3e81b49ae6ef0cf9022f90f3709166aa4488d
Parents: 0f51eae
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Aug 18 14:55:26 2016 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Thu Aug 18 14:55:26 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/security/SaslRpcClient.java | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5c3e81b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
index 850f27c..c360937 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
@@ -305,13 +305,16 @@ public class SaslRpcClient {
authType.getProtocol() + "/" + authType.getServerId(),
KerberosPrincipal.KRB_NT_SRV_HST).getName();
- boolean isPrincipalValid = false;
-
// use the pattern if defined
String serverKeyPattern = conf.get(serverKey + ".pattern");
if (serverKeyPattern != null && !serverKeyPattern.isEmpty()) {
Pattern pattern = GlobPattern.compile(serverKeyPattern);
- isPrincipalValid = pattern.matcher(serverPrincipal).matches();
+ if (!pattern.matcher(serverPrincipal).matches()) {
+ throw new IllegalArgumentException(String.format(
+ "Server has invalid Kerberos principal: %s,"
+ + " doesn't match the pattern: %s",
+ serverPrincipal, serverKeyPattern));
+ }
} else {
// check that the server advertised principal matches our conf
String confPrincipal = SecurityUtil.getServerPrincipal(
@@ -330,11 +333,11 @@ public class SaslRpcClient {
"Kerberos principal name does NOT have the expected hostname part: "
+ confPrincipal);
}
- isPrincipalValid = serverPrincipal.equals(confPrincipal);
- }
- if (!isPrincipalValid) {
- throw new IllegalArgumentException(
- "Server has invalid Kerberos principal: " + serverPrincipal);
+ if (!serverPrincipal.equals(confPrincipal)) {
+ throw new IllegalArgumentException(String.format(
+ "Server has invalid Kerberos principal: %s, expecting: %s",
+ serverPrincipal, confPrincipal));
+ }
}
return serverPrincipal;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDFS-10762. Pass IIP for file
status related methods. Contributed by Daryn Sharp.
Posted by cd...@apache.org.
HDFS-10762. Pass IIP for file status related methods. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22fc46d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22fc46d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22fc46d7
Branch: refs/heads/HDFS-9806
Commit: 22fc46d7659972ff016ccf1c6f781f0c160be26f
Parents: dc7a1c5
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Aug 22 15:37:02 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Aug 22 15:37:02 2016 -0500
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirAppendOp.java | 6 +-
.../server/namenode/FSDirStatAndListingOp.java | 80 +++++++++-----------
.../hdfs/server/namenode/FSDirWriteFileOp.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 14 ++--
.../hdfs/server/namenode/INodesInPath.java | 42 ++++++++--
.../hadoop/hdfs/TestReservedRawPaths.java | 21 +++++
6 files changed, 102 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 3a5d7dc..5192352 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -85,9 +85,10 @@ final class FSDirAppendOp {
final LocatedBlock lb;
final FSDirectory fsd = fsn.getFSDirectory();
final String src;
+ final INodesInPath iip;
fsd.writeLock();
try {
- final INodesInPath iip = fsd.resolvePathForWrite(pc, srcArg);
+ iip = fsd.resolvePathForWrite(pc, srcArg);
src = iip.getPath();
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
@@ -148,8 +149,7 @@ final class FSDirAppendOp {
fsd.writeUnlock();
}
- HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, src, false,
- FSDirectory.isReservedRawName(srcArg));
+ HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, iip);
if (lb != null) {
NameNode.stateChangeLog.debug(
"DIR* NameSystem.appendFile: file {} for {} at {} block {} block"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index c9eedf5..88be510 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -108,16 +108,16 @@ class FSDirStatAndListingOp {
if (!DFSUtil.isValidName(src)) {
throw new InvalidPathException("Invalid file name: " + src);
}
+ final INodesInPath iip;
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- final INodesInPath iip = fsd.resolvePath(pc, srcArg, resolveLink);
- src = iip.getPath();
+ iip = fsd.resolvePath(pc, srcArg, resolveLink);
fsd.checkPermission(pc, iip, false, null, null, null, null, false);
} else {
src = FSDirectory.resolvePath(srcArg, fsd);
+ iip = fsd.getINodesInPath(src, resolveLink);
}
- return getFileInfo(fsd, src, FSDirectory.isReservedRawName(srcArg),
- resolveLink);
+ return getFileInfo(fsd, iip);
}
/**
@@ -230,7 +230,6 @@ class FSDirStatAndListingOp {
String src, byte[] startAfter, boolean needLocation, boolean isSuperUser)
throws IOException {
String srcs = FSDirectory.normalizePath(src);
- final boolean isRawPath = FSDirectory.isReservedRawName(src);
if (FSDirectory.isExactReservedName(srcs)) {
return getReservedListing(fsd);
}
@@ -257,7 +256,7 @@ class FSDirStatAndListingOp {
return new DirectoryListing(
new HdfsFileStatus[]{ createFileStatus(
fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- needLocation, parentStoragePolicy, snapshot, isRawPath, iip)
+ needLocation, parentStoragePolicy, iip)
}, 0);
}
@@ -282,7 +281,7 @@ class FSDirStatAndListingOp {
cur.getLocalNameBytes());
listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), nodeAttrs,
needLocation, getStoragePolicyID(curPolicy, parentStoragePolicy),
- snapshot, isRawPath, iipWithChild);
+ iipWithChild);
listingCnt++;
if (needLocation) {
// Once we hit lsLimit locations, stop.
@@ -339,7 +338,6 @@ class FSDirStatAndListingOp {
listing[i] = createFileStatus(
fsd, sRoot.getLocalNameBytes(), nodeAttrs,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
- Snapshot.CURRENT_STATE_ID, false,
INodesInPath.fromINode(sRoot));
}
return new DirectoryListing(
@@ -363,10 +361,8 @@ class FSDirStatAndListingOp {
* @return object containing information regarding the file
* or null if file not found
*/
- static HdfsFileStatus getFileInfo(
- FSDirectory fsd, String path, INodesInPath iip, boolean isRawPath,
- boolean includeStoragePolicy)
- throws IOException {
+ static HdfsFileStatus getFileInfo(FSDirectory fsd,
+ INodesInPath iip, boolean includeStoragePolicy) throws IOException {
fsd.readLock();
try {
final INode node = iip.getLastINode();
@@ -377,36 +373,32 @@ class FSDirStatAndListingOp {
byte policyId = includeStoragePolicy && !node.isSymlink() ?
node.getStoragePolicyID() :
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
- INodeAttributes nodeAttrs = getINodeAttributes(fsd, path,
+ INodeAttributes nodeAttrs = getINodeAttributes(fsd, iip.getPath(),
HdfsFileStatus.EMPTY_NAME,
node, iip.getPathSnapshotId());
return createFileStatus(fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- policyId, iip.getPathSnapshotId(), isRawPath, iip);
+ policyId, iip);
} finally {
fsd.readUnlock();
}
}
- static HdfsFileStatus getFileInfo(
- FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath)
+ static HdfsFileStatus getFileInfo(FSDirectory fsd, INodesInPath iip)
throws IOException {
- fsd.readLock();
- try {
- HdfsFileStatus status = null;
- final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
- if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
- status = FSDirectory.DOT_RESERVED_STATUS;
- } else if (iip.isDotSnapshotDir()) {
- if (fsd.getINode4DotSnapshot(iip) != null) {
- status = FSDirectory.DOT_SNAPSHOT_DIR_STATUS;
- }
- } else {
- status = getFileInfo(fsd, src, iip, isRawPath, true);
+ if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
+ return FSDirectory.DOT_RESERVED_STATUS;
+ }
+
+ if (iip.isDotSnapshotDir()) {
+ if (fsd.getINode4DotSnapshot(iip) != null) {
+ return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
+ HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
+ HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
}
- return status;
- } finally {
- fsd.readUnlock();
+ return null;
}
+
+ return getFileInfo(fsd, iip, true);
}
/**
@@ -423,15 +415,12 @@ class FSDirStatAndListingOp {
*/
private static HdfsFileStatus createFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath,
- INodesInPath iip)
+ boolean needLocation, byte storagePolicy, INodesInPath iip)
throws IOException {
if (needLocation) {
- return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy,
- snapshot, isRawPath, iip);
+ return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
} else {
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
- snapshot, isRawPath, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
}
}
@@ -445,8 +434,7 @@ class FSDirStatAndListingOp {
INodesInPath iip) throws IOException {
INodeAttributes nodeAttrs = getINodeAttributes(
fsd, fullPath, path, iip.getLastINode(), snapshot);
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
- snapshot, isRawPath, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
}
/**
@@ -454,14 +442,15 @@ class FSDirStatAndListingOp {
* @param iip the INodesInPath containing the target INode and its ancestors
*/
static HdfsFileStatus createFileStatus(
- FSDirectory fsd, byte[] path,
- INodeAttributes nodeAttrs, byte storagePolicy, int snapshot,
- boolean isRawPath, INodesInPath iip) throws IOException {
+ FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
+ byte storagePolicy, INodesInPath iip) throws IOException {
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
final boolean isEncrypted;
final INode node = iip.getLastINode();
+ final int snapshot = iip.getPathSnapshotId();
+ final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
@@ -511,10 +500,9 @@ class FSDirStatAndListingOp {
* Create FileStatus with location info by file INode
* @param iip the INodesInPath containing the target INode and its ancestors
*/
- private static HdfsLocatedFileStatus createLocatedFileStatus(
+ private static HdfsFileStatus createLocatedFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- byte storagePolicy, int snapshot,
- boolean isRawPath, INodesInPath iip) throws IOException {
+ byte storagePolicy, INodesInPath iip) throws IOException {
assert fsd.hasReadLock();
long size = 0; // length is zero for directories
short replication = 0;
@@ -522,6 +510,8 @@ class FSDirStatAndListingOp {
LocatedBlocks loc = null;
final boolean isEncrypted;
final INode node = iip.getLastINode();
+ final int snapshot = iip.getPathSnapshotId();
+ final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 8429024..030d8cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -340,7 +340,6 @@ class FSDirWriteFileOp {
version = ezInfo.protocolVersion;
}
- boolean isRawPath = FSDirectory.isReservedRawName(src);
FSDirectory fsd = fsn.getFSDirectory();
INodesInPath iip = fsd.resolvePathForWrite(pc, src);
src = iip.getPath();
@@ -444,7 +443,7 @@ class FSDirWriteFileOp {
NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
src + " inode " + newNode.getId() + " " + holder);
}
- return FSDirStatAndListingOp.getFileInfo(fsd, src, false, isRawPath);
+ return FSDirStatAndListingOp.getFileInfo(fsd, iip);
}
static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index b6652e4..5dc5a03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -531,21 +531,24 @@ public class FSDirectory implements Closeable {
* @throws FileNotFoundException
* @throws AccessControlException
*/
- INodesInPath resolvePath(FSPermissionChecker pc, String src)
+ @VisibleForTesting
+ public INodesInPath resolvePath(FSPermissionChecker pc, String src)
throws UnresolvedLinkException, FileNotFoundException,
AccessControlException {
return resolvePath(pc, src, true);
}
- INodesInPath resolvePath(FSPermissionChecker pc, String src,
+ @VisibleForTesting
+ public INodesInPath resolvePath(FSPermissionChecker pc, String src,
boolean resolveLink) throws UnresolvedLinkException,
FileNotFoundException, AccessControlException {
byte[][] components = INode.getPathComponents(src);
- if (isPermissionEnabled && pc != null && isReservedRawName(components)) {
+ boolean isRaw = isReservedRawName(components);
+ if (isPermissionEnabled && pc != null && isRaw) {
pc.checkSuperuserPrivilege();
}
components = resolveComponents(components, this);
- return INodesInPath.resolve(rootDir, components, resolveLink);
+ return INodesInPath.resolve(rootDir, components, isRaw, resolveLink);
}
INodesInPath resolvePathForWrite(FSPermissionChecker pc, String src)
@@ -1662,8 +1665,7 @@ public class FSDirectory implements Closeable {
HdfsFileStatus getAuditFileInfo(INodesInPath iip)
throws IOException {
return (namesystem.isAuditEnabled() && namesystem.isExternalInvocation())
- ? FSDirStatAndListingOp.getFileInfo(this, iip.getPath(), iip, false,
- false) : null;
+ ? FSDirStatAndListingOp.getFileInfo(this, iip, false) : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index 86cab28..af8998f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -126,6 +126,12 @@ public class INodesInPath {
static INodesInPath resolve(final INodeDirectory startingDir,
final byte[][] components, final boolean resolveLink)
throws UnresolvedLinkException {
+ return resolve(startingDir, components, false, resolveLink);
+ }
+
+ static INodesInPath resolve(final INodeDirectory startingDir,
+ final byte[][] components, final boolean isRaw,
+ final boolean resolveLink) throws UnresolvedLinkException {
Preconditions.checkArgument(startingDir.compareTo(components[0]) == 0);
INode curNode = startingDir;
@@ -225,7 +231,7 @@ public class INodesInPath {
System.arraycopy(inodes, 0, newNodes, 0, newNodes.length);
inodes = newNodes;
}
- return new INodesInPath(inodes, components, isSnapshot, snapshotId);
+ return new INodesInPath(inodes, components, isRaw, isSnapshot, snapshotId);
}
private static boolean shouldUpdateLatestId(int sid, int snapshotId) {
@@ -249,7 +255,8 @@ public class INodesInPath {
INode[] inodes = new INode[iip.inodes.length];
System.arraycopy(iip.inodes, 0, inodes, 0, inodes.length);
inodes[pos] = inode;
- return new INodesInPath(inodes, iip.path, iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, iip.path, iip.isRaw,
+ iip.isSnapshot, iip.snapshotId);
}
/**
@@ -267,7 +274,8 @@ public class INodesInPath {
byte[][] path = new byte[iip.path.length + 1][];
System.arraycopy(iip.path, 0, path, 0, path.length - 1);
path[path.length - 1] = childName;
- return new INodesInPath(inodes, path, iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, path, iip.isRaw,
+ iip.isSnapshot, iip.snapshotId);
}
private final byte[][] path;
@@ -279,6 +287,13 @@ public class INodesInPath {
* true if this path corresponds to a snapshot
*/
private final boolean isSnapshot;
+
+ /**
+ * true if this is a /.reserved/raw path. path component resolution strips
+ * it from the path so need to track it separately.
+ */
+ private final boolean isRaw;
+
/**
* For snapshot paths, it is the id of the snapshot; or
* {@link Snapshot#CURRENT_STATE_ID} if the snapshot does not exist. For
@@ -287,17 +302,18 @@ public class INodesInPath {
*/
private final int snapshotId;
- private INodesInPath(INode[] inodes, byte[][] path, boolean isSnapshot,
- int snapshotId) {
+ private INodesInPath(INode[] inodes, byte[][] path, boolean isRaw,
+ boolean isSnapshot,int snapshotId) {
Preconditions.checkArgument(inodes != null && path != null);
this.inodes = inodes;
this.path = path;
+ this.isRaw = isRaw;
this.isSnapshot = isSnapshot;
this.snapshotId = snapshotId;
}
private INodesInPath(INode[] inodes, byte[][] path) {
- this(inodes, path, false, CURRENT_STATE_ID);
+ this(inodes, path, false, false, CURRENT_STATE_ID);
}
/**
@@ -400,7 +416,7 @@ public class INodesInPath {
final byte[][] apath = new byte[length][];
System.arraycopy(this.inodes, 0, anodes, 0, length);
System.arraycopy(this.path, 0, apath, 0, length);
- return new INodesInPath(anodes, apath, false, snapshotId);
+ return new INodesInPath(anodes, apath, isRaw, false, snapshotId);
}
/**
@@ -428,7 +444,7 @@ public class INodesInPath {
byte[][] existingPath = new byte[i][];
System.arraycopy(inodes, 0, existing, 0, i);
System.arraycopy(path, 0, existingPath, 0, i);
- return new INodesInPath(existing, existingPath, false, snapshotId);
+ return new INodesInPath(existing, existingPath, isRaw, false, snapshotId);
}
/**
@@ -438,10 +454,20 @@ public class INodesInPath {
return this.isSnapshot;
}
+ /**
+ * @return if .snapshot is the last path component.
+ */
boolean isDotSnapshotDir() {
return isDotSnapshotDir(getLastLocalName());
}
+ /**
+ * @return if this is a /.reserved/raw path.
+ */
+ public boolean isRaw() {
+ return isRaw;
+ }
+
private static String toString(INode inode) {
return inode == null? null: inode.getLocalName();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
index 449f715..c09d346 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
+import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
@@ -49,6 +50,8 @@ import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesNotEqual;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.apache.hadoop.test.GenericTestUtils.assertMatches;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestReservedRawPaths {
@@ -100,6 +103,24 @@ public class TestReservedRawPaths {
}
/**
+ * Verify resolving path will return an iip that tracks if the original
+ * path was a raw path.
+ */
+ @Test(timeout = 120000)
+ public void testINodesInPath() throws IOException {
+ FSDirectory fsd = cluster.getNamesystem().getFSDirectory();
+ final String path = "/path";
+
+ INodesInPath iip = fsd.resolvePath(null, path);
+ assertFalse(iip.isRaw());
+ assertEquals(path, iip.getPath());
+
+ iip = fsd.resolvePath(null, "/.reserved/raw" + path);
+ assertTrue(iip.isRaw());
+ assertEquals(path, iip.getPath());
+ }
+
+ /**
* Basic read/write tests of raw files.
* Create a non-encrypted file
* Create an encryption zone
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HADOOP-13487. Hadoop KMS should
load old delegation tokens from Zookeeper on startup. Contributed by Xiao
Chen.
Posted by cd...@apache.org.
HADOOP-13487. Hadoop KMS should load old delegation tokens from Zookeeper on startup. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4d4d347
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4d4d347
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4d4d347
Branch: refs/heads/HDFS-9806
Commit: f4d4d3474cfd2d1f2d243f5ae5cec17af38270b1
Parents: 22fc46d
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Aug 22 14:31:13 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Aug 22 14:42:13 2016 -0700
----------------------------------------------------------------------
.../ZKDelegationTokenSecretManager.java | 44 +++++++++
.../TestZKDelegationTokenSecretManager.java | 93 +++++++++++++++++++-
2 files changed, 136 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d4d347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
index c3ad9f3..6c66e98 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
@@ -361,6 +361,7 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
}
}
}, listenerThreadPool);
+ loadFromZKCache(false);
}
} catch (Exception e) {
throw new IOException("Could not start PathChildrenCache for keys", e);
@@ -389,6 +390,7 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
}
}
}, listenerThreadPool);
+ loadFromZKCache(true);
}
} catch (Exception e) {
throw new IOException("Could not start PathChildrenCache for tokens", e);
@@ -396,6 +398,43 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
super.startThreads();
}
+ /**
+ * Load the PathChildrenCache into the in-memory map. Possible caches to be
+ * loaded are keyCache and tokenCache.
+ *
+ * @param isTokenCache true if loading tokenCache, false if loading keyCache.
+ */
+ private void loadFromZKCache(final boolean isTokenCache) {
+ final String cacheName = isTokenCache ? "token" : "key";
+ LOG.info("Starting to load {} cache.", cacheName);
+ final List<ChildData> children;
+ if (isTokenCache) {
+ children = tokenCache.getCurrentData();
+ } else {
+ children = keyCache.getCurrentData();
+ }
+
+ int count = 0;
+ for (ChildData child : children) {
+ try {
+ if (isTokenCache) {
+ processTokenAddOrUpdate(child);
+ } else {
+ processKeyAddOrUpdate(child.getData());
+ }
+ } catch (Exception e) {
+ LOG.info("Ignoring node {} because it failed to load.",
+ child.getPath());
+ LOG.debug("Failure exception:", e);
+ ++count;
+ }
+ }
+ if (count > 0) {
+ LOG.warn("Ignored {} nodes while loading {} cache.", count, cacheName);
+ }
+ LOG.info("Loaded {} cache.", cacheName);
+ }
+
private void processKeyAddOrUpdate(byte[] data) throws IOException {
ByteArrayInputStream bin = new ByteArrayInputStream(data);
DataInputStream din = new DataInputStream(bin);
@@ -890,4 +929,9 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
public ExecutorService getListenerThreadPool() {
return listenerThreadPool;
}
+
+ @VisibleForTesting
+ DelegationTokenInformation getTokenInfoFromMemory(TokenIdent ident) {
+ return currentTokens.get(ident);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d4d347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
index 185a994..c9571ff2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
+import com.google.common.base.Supplier;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -37,6 +38,7 @@ import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
@@ -44,12 +46,18 @@ import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.junit.Assert.fail;
-import org.junit.Test;
public class TestZKDelegationTokenSecretManager {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestZKDelegationTokenSecretManager.class);
private static final int TEST_RETRIES = 2;
@@ -61,6 +69,9 @@ public class TestZKDelegationTokenSecretManager {
private TestingServer zkServer;
+ @Rule
+ public Timeout globalTimeout = new Timeout(300000);
+
@Before
public void setup() throws Exception {
zkServer = new TestingServer();
@@ -382,4 +393,84 @@ public class TestZKDelegationTokenSecretManager {
}
}
+ @SuppressWarnings({ "unchecked" })
+ @Test
+ public void testNodesLoadedAfterRestart() throws Exception {
+ final String connectString = zkServer.getConnectString();
+ final Configuration conf = getSecretConf(connectString);
+ final int removeScan = 1;
+ // Set the remove scan interval to remove expired tokens
+ conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, removeScan);
+ // Set the update interval to trigger background thread to run. The thread
+ // is hard-coded to sleep at least 5 seconds.
+ conf.setLong(DelegationTokenManager.UPDATE_INTERVAL, 5);
+ // Set token expire time to 5 seconds.
+ conf.setLong(DelegationTokenManager.RENEW_INTERVAL, 5);
+
+ DelegationTokenManager tm =
+ new DelegationTokenManager(conf, new Text("bla"));
+ tm.init();
+ Token<DelegationTokenIdentifier> token =
+ (Token<DelegationTokenIdentifier>) tm
+ .createToken(UserGroupInformation.getCurrentUser(), "good");
+ Assert.assertNotNull(token);
+ Token<DelegationTokenIdentifier> cancelled =
+ (Token<DelegationTokenIdentifier>) tm
+ .createToken(UserGroupInformation.getCurrentUser(), "cancelled");
+ Assert.assertNotNull(cancelled);
+ tm.verifyToken(token);
+ tm.verifyToken(cancelled);
+
+ // Cancel one token, verify it's gone
+ tm.cancelToken(cancelled, "cancelled");
+ final AbstractDelegationTokenSecretManager sm =
+ tm.getDelegationTokenSecretManager();
+ final ZKDelegationTokenSecretManager zksm =
+ (ZKDelegationTokenSecretManager) sm;
+ final AbstractDelegationTokenIdentifier idCancelled =
+ sm.decodeTokenIdentifier(cancelled);
+ LOG.info("Waiting for the cancelled token to be removed");
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ AbstractDelegationTokenSecretManager.DelegationTokenInformation dtinfo =
+ zksm.getTokenInfo(idCancelled);
+ return dtinfo == null;
+ }
+ }, 100, 5000);
+
+ // Fake a restart which launches a new tm
+ tm.destroy();
+ tm = new DelegationTokenManager(conf, new Text("bla"));
+ tm.init();
+ final AbstractDelegationTokenSecretManager smNew =
+ tm.getDelegationTokenSecretManager();
+ final ZKDelegationTokenSecretManager zksmNew =
+ (ZKDelegationTokenSecretManager) smNew;
+
+ // The cancelled token should be gone, and not loaded.
+ AbstractDelegationTokenIdentifier id =
+ smNew.decodeTokenIdentifier(cancelled);
+ AbstractDelegationTokenSecretManager.DelegationTokenInformation dtinfo =
+ zksmNew.getTokenInfo(id);
+ Assert.assertNull("canceled dt should be gone!", dtinfo);
+
+ // The good token should be loaded on startup, and removed after expiry.
+ id = smNew.decodeTokenIdentifier(token);
+ dtinfo = zksmNew.getTokenInfoFromMemory(id);
+ Assert.assertNotNull("good dt should be in memory!", dtinfo);
+
+ // Wait for the good token to expire.
+ Thread.sleep(5000);
+ final ZKDelegationTokenSecretManager zksm1 = zksmNew;
+ final AbstractDelegationTokenIdentifier id1 = id;
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ LOG.info("Waiting for the expired token to be removed...");
+ return zksm1.getTokenInfo(id1) == null;
+ }
+ }, 1000, 5000);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDFS-10692. Update JDiff report's
base version for HDFS from 2.6.0 to 2.7.2. Contributed by Wangda Tan.
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc7a1c54/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.7.2.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.7.2.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.7.2.xml
new file mode 100644
index 0000000..028ba2d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.7.2.xml
@@ -0,0 +1,21505 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Thu Aug 18 16:02:32 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="Apache Hadoop HDFS 2.7.2"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/classes:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.7.2.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/lib/tools.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.7.2.jar:/Users/wtan/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar:/Users/wtan/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/wtan/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/http
core-4.2.5.jar:/Users/wtan/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/wtan/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/wtan/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/wtan/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/wtan/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/wtan/.m2/repository/jline/jline/0.9.94/jline-0.9.94.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/hadoop-common-2.7.2.jar:/Users/wtan/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/wtan/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/
wtan/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/wtan/.m2/repository/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/Users/wtan/.m2/repository/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/wtan/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/wtan/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/wtan/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/wtan/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/wtan/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/wtan/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/wtan/.m2/repos
itory/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/wtan/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/wtan/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/wtan/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/wtan/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/wtan/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/wtan/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/wtan/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/wtan/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/wtan/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar:/Users/wtan/.m2/repository/org/apache/curator/curat
or-recipes/2.7.1/curator-recipes-2.7.1.jar:/Users/wtan/.m2/repository/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar:/Users/wtan/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/wtan/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/wtan/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/wtan/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/wtan/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/wtan/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/wtan/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/wtan/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/wtan/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/wtan/.m2/repository/commons-lang/common
s-lang/2.6/commons-lang-2.6.jar:/Users/wtan/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/wtan/.m2/repository/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/Users/wtan/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/wtan/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/wtan/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/wtan/.m2/repository/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/wtan/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/wtan/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/wtan/.m2/repository/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar:/Users/wtan/.m2/repository/xerces/xercesIm
pl/2.9.1/xercesImpl-2.9.1.jar:/Users/wtan/.m2/repository/xml-apis/xml-apis/1.3.04/xml-apis-1.3.04.jar:/Users/wtan/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar:/Users/wtan/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar -sourcepath /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -apidir /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 2.7.2 -->
+<package name="org.apache.hadoop.fs">
+ <!-- start class org.apache.hadoop.fs.BlockStorageLocation -->
+ <class name="BlockStorageLocation" extends="org.apache.hadoop.fs.BlockLocation"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="BlockStorageLocation" type="org.apache.hadoop.fs.BlockLocation, org.apache.hadoop.fs.VolumeId[]"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getVolumeIds" return="org.apache.hadoop.fs.VolumeId[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets the list of {@link VolumeId} corresponding to the block's replicas.
+
+ @return volumeIds list of VolumeId for the block's replicas]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Wrapper for {@link BlockLocation} that also adds {@link VolumeId} volume
+ location information for each replica.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.fs.BlockStorageLocation -->
+ <!-- start class org.apache.hadoop.fs.CacheFlag -->
+ <class name="CacheFlag" extends="java.lang.Enum"
+ abstract="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.fs.CacheFlag[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.fs.CacheFlag"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ <doc>
+ <![CDATA[Specifies semantics for CacheDirective operations. Multiple flags can
+ be combined in an EnumSet.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.fs.CacheFlag -->
+ <!-- start class org.apache.hadoop.fs.HdfsVolumeId -->
+ <class name="HdfsVolumeId" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.fs.VolumeId"/>
+ <constructor name="HdfsVolumeId" type="byte[]"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="compareTo" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.fs.VolumeId"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="obj" type="java.lang.Object"/>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[HDFS-specific volume identifier which implements {@link VolumeId}. Can be
+ used to differentiate between the data directories on a single datanode. This
+ identifier is only unique on a per-datanode basis.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.fs.HdfsVolumeId -->
+ <!-- start interface org.apache.hadoop.fs.VolumeId -->
+ <interface name="VolumeId" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="java.lang.Comparable"/>
+ <method name="compareTo" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.fs.VolumeId"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="obj" type="java.lang.Object"/>
+ </method>
+ <doc>
+ <![CDATA[Opaque interface that identifies a disk location. Subclasses
+ should implement {@link Comparable} and override both equals and hashCode.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.fs.VolumeId -->
+ <!-- start class org.apache.hadoop.fs.XAttr.Builder -->
+ <class name="XAttr.Builder" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Builder"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setNameSpace" return="org.apache.hadoop.fs.XAttr.Builder"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ns" type="org.apache.hadoop.fs.XAttr.NameSpace"/>
+ </method>
+ <method name="setName" return="org.apache.hadoop.fs.XAttr.Builder"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ <method name="setValue" return="org.apache.hadoop.fs.XAttr.Builder"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="value" type="byte[]"/>
+ </method>
+ <method name="build" return="org.apache.hadoop.fs.XAttr"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.fs.XAttr.Builder -->
+ <!-- start class org.apache.hadoop.fs.XAttr.NameSpace -->
+ <class name="XAttr.NameSpace" extends="java.lang.Enum"
+ abstract="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.fs.XAttr.NameSpace[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.fs.XAttr.NameSpace"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.fs.XAttr.NameSpace -->
+</package>
+<package name="org.apache.hadoop.hdfs">
+ <!-- start interface org.apache.hadoop.hdfs.BlockReader -->
+ <interface name="BlockReader" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.fs.ByteBufferReadable"/>
+ <method name="read" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="buf" type="byte[]"/>
+ <param name="off" type="int"/>
+ <param name="len" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="skip" return="long"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="n" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Skip the given number of bytes]]>
+ </doc>
+ </method>
+ <method name="available" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns an estimate of the number of bytes that can be read
+ (or skipped over) from this input stream without performing
+ network I/O.
+ This may return more than what is actually present in the block.]]>
+ </doc>
+ </method>
+ <method name="close"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Close the block reader.
+
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="readFully"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="buf" type="byte[]"/>
+ <param name="readOffset" type="int"/>
+ <param name="amtToRead" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Read exactly the given amount of data, throwing an exception
+ if EOF is reached before that amount]]>
+ </doc>
+ </method>
+ <method name="readAll" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="buf" type="byte[]"/>
+ <param name="offset" type="int"/>
+ <param name="len" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Similar to {@link #readFully(byte[], int, int)} except that it will
+ not throw an exception on EOF. However, it differs from the simple
+ {@link #read(byte[], int, int)} call in that it is guaranteed to
+ read the data if it is available. In other words, if this call
+ does not throw an exception, then either the buffer has been
+ filled or the next call will return EOF.]]>
+ </doc>
+ </method>
+ <method name="isLocal" return="boolean"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return true only if this is a local read.]]>
+ </doc>
+ </method>
+ <method name="isShortCircuit" return="boolean"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return true only if this is a short-circuit read.
+ All short-circuit reads are also local.]]>
+ </doc>
+ </method>
+ <method name="getClientMmap" return="org.apache.hadoop.hdfs.shortcircuit.ClientMmap"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="opts" type="java.util.EnumSet"/>
+ <doc>
+ <![CDATA[Get a ClientMmap object for this BlockReader.
+
+ @param opts The read options to use.
+ @return The ClientMmap object, or null if mmap is not
+ supported.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[A BlockReader is responsible for reading a single block
+ from a single datanode.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.hdfs.BlockReader -->
+ <!-- start class org.apache.hadoop.hdfs.BlockReaderFactory.BlockReaderPeer -->
+ <class name="BlockReaderFactory.BlockReaderPeer" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.BlockReaderFactory.BlockReaderPeer -->
+ <!-- start class org.apache.hadoop.hdfs.BlockReaderFactory.FailureInjector -->
+ <class name="BlockReaderFactory.FailureInjector" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="FailureInjector"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="injectRequestFileDescriptorsFailure"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getSupportsReceiptVerification" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.BlockReaderFactory.FailureInjector -->
+ <!-- start class org.apache.hadoop.hdfs.CorruptFileBlockIterator -->
+ <class name="CorruptFileBlockIterator" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.fs.RemoteIterator"/>
+ <constructor name="CorruptFileBlockIterator" type="org.apache.hadoop.hdfs.DFSClient, org.apache.hadoop.fs.Path"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getCallsMade" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return the number of calls made to the DFSClient.
+ This is for debugging and testing purposes.]]>
+ </doc>
+ </method>
+ <method name="hasNext" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="next" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <doc>
+ <![CDATA[Provides an iterator interface for listCorruptFileBlocks.
+ This class is used by DistributedFileSystem and Hdfs.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.CorruptFileBlockIterator -->
+ <!-- start class org.apache.hadoop.hdfs.DFSClient.Conf -->
+ <class name="DFSClient.Conf" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Conf" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="isUseLegacyBlockReaderLocal" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDomainSocketPath" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="isShortCircuitLocalReads" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="isDomainSocketDataTraffic" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <field name="brfFailureInjector" type="org.apache.hadoop.hdfs.BlockReaderFactory.FailureInjector"
+ transient="false" volatile="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[DFSClient configuration]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSClient.Conf -->
+ <!-- start class org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream -->
+ <class name="DFSClient.DFSDataInputStream" extends="org.apache.hadoop.hdfs.client.HdfsDataInputStream"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="use {@link HdfsDataInputStream} instead.">
+ <constructor name="DFSDataInputStream" type="org.apache.hadoop.hdfs.DFSInputStream"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <doc>
+ <![CDATA[@deprecated use {@link HdfsDataInputStream} instead.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream -->
+ <!-- start class org.apache.hadoop.hdfs.DFSHedgedReadMetrics -->
+ <class name="DFSHedgedReadMetrics" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="DFSHedgedReadMetrics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="incHedgedReadOps"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="incHedgedReadOpsInCurThread"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="incHedgedReadWins"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getHedgedReadOps" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getHedgedReadOpsInCurThread" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getHedgedReadWins" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <field name="hedgedReadOps" type="java.util.concurrent.atomic.AtomicLong"
+ transient="false" volatile="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="hedgedReadOpsWin" type="java.util.concurrent.atomic.AtomicLong"
+ transient="false" volatile="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="hedgedReadOpsInCurThread" type="java.util.concurrent.atomic.AtomicLong"
+ transient="false" volatile="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[The client-side metrics for hedged read feature.
+ This class has a number of metrics variables that are publicly accessible,
+ we can grab them from client side, like HBase.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSHedgedReadMetrics -->
+ <!-- start class org.apache.hadoop.hdfs.DFSInotifyEventInputStream -->
+ <class name="DFSInotifyEventInputStream" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="poll" return="org.apache.hadoop.hdfs.inotify.EventBatch"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="MissingEventsException" type="org.apache.hadoop.hdfs.inotify.MissingEventsException"/>
+ <doc>
+ <![CDATA[Returns the next batch of events in the stream or null if no new
+ batches are currently available.
+
+ @throws IOException because of network error or edit log
+ corruption. Also possible if JournalNodes are unresponsive in the
+ QJM setting (even one unresponsive JournalNode is enough in rare cases),
+ so catching this exception and retrying at least a few times is
+ recommended.
+ @throws MissingEventsException if we cannot return the next batch in the
+ stream because the data for the events (and possibly some subsequent
+ events) has been deleted (generally because this stream is a very large
+ number of transactions behind the current state of the NameNode). It is
+ safe to continue reading from the stream after this exception is thrown
+ The next available batch of events will be returned.]]>
+ </doc>
+ </method>
+ <method name="getTxidsBehindEstimate" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return a estimate of how many transaction IDs behind the NameNode's
+ current state this stream is. Clients should periodically call this method
+ and check if its result is steadily increasing, which indicates that they
+ are falling behind (i.e. transaction are being generated faster than the
+ client is reading them). If a client falls too far behind events may be
+ deleted before the client can read them.
+ <p/>
+ A return value of -1 indicates that an estimate could not be produced, and
+ should be ignored. The value returned by this method is really only useful
+ when compared to previous or subsequent returned values.]]>
+ </doc>
+ </method>
+ <method name="poll" return="org.apache.hadoop.hdfs.inotify.EventBatch"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="time" type="long"/>
+ <param name="tu" type="java.util.concurrent.TimeUnit"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ <exception name="MissingEventsException" type="org.apache.hadoop.hdfs.inotify.MissingEventsException"/>
+ <doc>
+ <![CDATA[Returns the next event batch in the stream, waiting up to the specified
+ amount of time for a new batch. Returns null if one is not available at the
+ end of the specified amount of time. The time before the method returns may
+ exceed the specified amount of time by up to the time required for an RPC
+ to the NameNode.
+
+ @param time number of units of the given TimeUnit to wait
+ @param tu the desired TimeUnit
+ @throws IOException see {@link DFSInotifyEventInputStream#poll()}
+ @throws MissingEventsException
+ see {@link DFSInotifyEventInputStream#poll()}
+ @throws InterruptedException if the calling thread is interrupted]]>
+ </doc>
+ </method>
+ <method name="take" return="org.apache.hadoop.hdfs.inotify.EventBatch"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ <exception name="MissingEventsException" type="org.apache.hadoop.hdfs.inotify.MissingEventsException"/>
+ <doc>
+ <![CDATA[Returns the next batch of events in the stream, waiting indefinitely if
+ a new batch is not immediately available.
+
+ @throws IOException see {@link DFSInotifyEventInputStream#poll()}
+ @throws MissingEventsException see
+ {@link DFSInotifyEventInputStream#poll()}
+ @throws InterruptedException if the calling thread is interrupted]]>
+ </doc>
+ </method>
+ <field name="LOG" type="org.slf4j.Logger"
+ transient="false" volatile="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[Stream for reading inotify events. DFSInotifyEventInputStreams should not
+ be shared among multiple threads.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSInotifyEventInputStream -->
+ <!-- start class org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics -->
+ <class name="DFSInputStream.ReadStatistics" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ReadStatistics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="ReadStatistics" type="org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getTotalBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total bytes read. This will always be at least as
+ high as the other numbers, since it includes all of them.]]>
+ </doc>
+ </method>
+ <method name="getTotalLocalBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total local bytes read. This will always be at least
+ as high as totalShortCircuitBytesRead, since all short-circuit
+ reads are also local.]]>
+ </doc>
+ </method>
+ <method name="getTotalShortCircuitBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total short-circuit local bytes read.]]>
+ </doc>
+ </method>
+ <method name="getTotalZeroCopyBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total number of zero-copy bytes read.]]>
+ </doc>
+ </method>
+ <method name="getRemoteBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total number of bytes read which were not local.]]>
+ </doc>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics -->
+ <!-- start class org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress -->
+ <class name="DFSUtil.ConfiguredNNAddress" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getNameserviceId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getNamenodeId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getAddress" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Represent one of the NameNodes configured in the cluster.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress -->
+ <!-- start class org.apache.hadoop.hdfs.ExtendedBlockId -->
+ <class name="ExtendedBlockId" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ExtendedBlockId" type="long, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="fromExtendedBlock" return="org.apache.hadoop.hdfs.ExtendedBlockId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="block" type="org.apache.hadoop.hdfs.protocol.ExtendedBlock"/>
+ </method>
+ <method name="getBlockId" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getBlockPoolId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="o" type="java.lang.Object"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[An immutable key which identifies a block.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.ExtendedBlockId -->
+ <!-- start class org.apache.hadoop.hdfs.HAUtil -->
+ <class name="HAUtil" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="isHAEnabled" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Returns true if HA for namenode is configured for the given nameservice
+
+ @param conf Configuration
+ @param nsId nameservice, or null if no federated NS is configured
+ @return true if HA is configured in the configuration; else false.]]>
+ </doc>
+ </method>
+ <method name="usesSharedEditsDir" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Returns true if HA is using a shared edits directory.
+
+ @param conf Configuration
+ @return true if HA config is using a shared edits dir, false otherwise.]]>
+ </doc>
+ </method>
+ <method name="getNameNodeId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the namenode Id by matching the {@code addressKey}
+ with the the address of the local node.
+
+ If {@link DFSConfigKeys#DFS_HA_NAMENODE_ID_KEY} is not specifically
+ configured, this method determines the namenode Id by matching the local
+ node's address with the configured addresses. When a match is found, it
+ returns the namenode Id from the corresponding configuration key.
+
+ @param conf Configuration
+ @return namenode Id on success, null on failure.
+ @throws HadoopIllegalArgumentException on error]]>
+ </doc>
+ </method>
+ <method name="getNameNodeIdFromAddress" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="address" type="java.net.InetSocketAddress"/>
+ <param name="keys" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Similar to
+ {@link DFSUtil#getNameServiceIdFromAddress(Configuration,
+ InetSocketAddress, String...)}]]>
+ </doc>
+ </method>
+ <method name="getNameNodeIdOfOtherNode" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the NN ID of the other node in an HA setup.
+
+ @param conf the configuration of this node
+ @return the NN ID of the other node in this nameservice]]>
+ </doc>
+ </method>
+ <method name="getConfForOtherNode" return="org.apache.hadoop.conf.Configuration"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="myConf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Given the configuration for this node, return a Configuration object for
+ the other node in an HA setup.
+
+ @param myConf the configuration of this node
+ @return the configuration of the other node in an HA setup]]>
+ </doc>
+ </method>
+ <method name="shouldAllowStandbyReads" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[This is used only by tests at the moment.
+ @return true if the NN should allow read operations while in standby mode.]]>
+ </doc>
+ </method>
+ <method name="setAllowStandbyReads"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="val" type="boolean"/>
+ </method>
+ <method name="isLogicalUri" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <doc>
+ <![CDATA[@return true if the given nameNodeUri appears to be a logical URI.]]>
+ </doc>
+ </method>
+ <method name="isClientFailoverConfigured" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <doc>
+ <![CDATA[Check whether the client has a failover proxy provider configured
+ for the namenode/nameservice.
+
+ @param conf Configuration
+ @param nameNodeUri The URI of namenode
+ @return true if failover is configured.]]>
+ </doc>
+ </method>
+ <method name="useLogicalUri" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Check whether logical URI is needed for the namenode and
+ the corresponding failover proxy provider in the config.
+
+ @param conf Configuration
+ @param nameNodeUri The URI of namenode
+ @return true if logical URI is needed. false, if not needed.
+ @throws IOException most likely due to misconfiguration.]]>
+ </doc>
+ </method>
+ <method name="getServiceUriFromToken" return="java.net.URI"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="scheme" type="java.lang.String"/>
+ <param name="token" type="org.apache.hadoop.security.token.Token"/>
+ <doc>
+ <![CDATA[Parse the file system URI out of the provided token.]]>
+ </doc>
+ </method>
+ <method name="buildTokenServiceForLogicalUri" return="org.apache.hadoop.io.Text"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="uri" type="java.net.URI"/>
+ <param name="scheme" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the service name used in the delegation token for the given logical
+ HA service.
+ @param uri the logical URI of the cluster
+ @param scheme the scheme of the corresponding FileSystem
+ @return the service name]]>
+ </doc>
+ </method>
+ <method name="isTokenForLogicalUri" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="token" type="org.apache.hadoop.security.token.Token"/>
+ <doc>
+ <![CDATA[@return true if this token corresponds to a logical nameservice
+ rather than a specific namenode.]]>
+ </doc>
+ </method>
+ <method name="buildTokenServicePrefixForLogicalUri" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="scheme" type="java.lang.String"/>
+ </method>
+ <method name="cloneDelegationTokenForLogicalUri"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="haUri" type="java.net.URI"/>
+ <param name="nnAddrs" type="java.util.Collection"/>
+ <doc>
+ <![CDATA[Locate a delegation token associated with the given HA cluster URI, and if
+ one is found, clone it to also represent the underlying namenode address.
+ @param ugi the UGI to modify
+ @param haUri the logical URI for the cluster
+ @param nnAddrs collection of NNs in the cluster to which the token
+ applies]]>
+ </doc>
+ </method>
+ <method name="getAddressOfActive" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the internet address of the currently-active NN. This should rarely be
+ used, since callers of this method who connect directly to the NN using the
+ resulting InetSocketAddress will not be able to connect to the active NN if
+ a failover were to occur after this method has been called.
+
+ @param fs the file system to get the active address of.
+ @return the internet address of the currently-active NN.
+ @throws IOException if an error occurs while resolving the active NN.]]>
+ </doc>
+ </method>
+ <method name="getProxiesForAllNameNodesInNameservice" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
+ call should be made on every NN in an HA nameservice, not just the active.
+
+ @param conf configuration
+ @param nsId the nameservice to get all of the proxies for.
+ @return a list of RPC proxies for each NN in the nameservice.
+ @throws IOException in the event of error.]]>
+ </doc>
+ </method>
+ <method name="getProxiesForAllNameNodesInNameservice" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <param name="xface" type="java.lang.Class"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
+ call should be made on every NN in an HA nameservice, not just the active.
+
+ @param conf configuration
+ @param nsId the nameservice to get all of the proxies for.
+ @param xface the protocol class.
+ @return a list of RPC proxies for each NN in the nameservice.
+ @throws IOException in the event of error.]]>
+ </doc>
+ </method>
+ <method name="isAtLeastOneActive" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="namenodes" type="java.util.List"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Used to ensure that at least one of the given HA NNs is currently in the
+ active state..
+
+ @param namenodes list of RPC proxies for each NN to check.
+ @return true if at least one NN is active, false if all are in the standby state.
+ @throws IOException in the event of error.]]>
+ </doc>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.HAUtil -->
+ <!-- start class org.apache.hadoop.hdfs.KeyProviderCache -->
+ <class name="KeyProviderCache" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="KeyProviderCache" type="long"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="get" return="org.apache.hadoop.crypto.key.KeyProvider"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="setKeyProvider"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="keyProvider" type="org.apache.hadoop.crypto.key.KeyProvider"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <field name="LOG" type="org.apache.commons.logging.Log"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.KeyProviderCache -->
+ <!-- start class org.apache.hadoop.hdfs.NameNodeProxies -->
+ <class name="NameNodeProxies" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="NameNodeProxies"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="createProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates the namenode proxy with the passed protocol. This will handle
+ creation of either HA- or non-HA-enabled proxy objects, depending upon
+ if the provided URI is a configured logical URI.
+
+ @param conf the configuration containing the required IPC
+ properties, client failover configurations, etc.
+ @param nameNodeUri the URI pointing either to a specific NameNode
+ or to a logical nameservice.
+ @param xface the IPC interface which should be created
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException if there is an error creating the proxy]]>
+ </doc>
+ </method>
+ <method name="createProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates the namenode proxy with the passed protocol. This will handle
+ creation of either HA- or non-HA-enabled proxy objects, depending upon
+ if the provided URI is a configured logical URI.
+
+ @param conf the configuration containing the required IPC
+ properties, client failover configurations, etc.
+ @param nameNodeUri the URI pointing either to a specific NameNode
+ or to a logical nameservice.
+ @param xface the IPC interface which should be created
+ @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ a secure client falls back to simple auth
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException if there is an error creating the proxy]]>
+ </doc>
+ </method>
+ <method name="createProxyWithLossyRetryHandler" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="numResponseToDrop" type="int"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Generate a dummy namenode proxy instance that utilizes our hacked
+ {@link LossyRetryInvocationHandler}. Proxy instance generated using this
+ method will proactively drop RPC responses. Currently this method only
+ support HA setup. null will be returned if the given configuration is not
+ for HA.
+
+ @param config the configuration containing the required IPC
+ properties, client failover configurations, etc.
+ @param nameNodeUri the URI pointing either to a specific NameNode
+ or to a logical nameservice.
+ @param xface the IPC interface which should be created
+ @param numResponseToDrop The number of responses to drop for each RPC call
+ @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ a secure client falls back to simple auth
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to. Will return null of the
+ given configuration does not support HA.
+ @throws IOException if there is an error creating the proxy]]>
+ </doc>
+ </method>
+ <method name="createNonHAProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nnAddr" type="java.net.InetSocketAddress"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="withRetries" type="boolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates an explicitly non-HA-enabled proxy object. Most of the time you
+ don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
+
+ @param conf the configuration object
+ @param nnAddr address of the remote NN to connect to
+ @param xface the IPC interface which should be created
+ @param ugi the user who is making the calls on the proxy object
+ @param withRetries certain interfaces have a non-standard retry policy
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="createNonHAProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nnAddr" type="java.net.InetSocketAddress"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="withRetries" type="boolean"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates an explicitly non-HA-enabled proxy object. Most of the time you
+ don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
+
+ @param conf the configuration object
+ @param nnAddr address of the remote NN to connect to
+ @param xface the IPC interface which should be created
+ @param ugi the user who is making the calls on the proxy object
+ @param withRetries certain interfaces have a non-standard retry policy
+ @param fallbackToSimpleAuth - set to true or false during this method to
+ indicate if a secure client falls back to simple auth
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getFailoverProxyProviderClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Gets the configured Failover proxy provider's class]]>
+ </doc>
+ </method>
+ <method name="createFailoverProxyProvider" return="org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="checkPort" type="boolean"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates the Failover proxy provider instance]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Create proxy objects to communicate with a remote NN. All remote access to an
+ NN should be funneled through this class. Most of the time you'll want to use
+ {@link NameNodeProxies#createProxy(Configuration, URI, Class)}, which will
+ create either an HA- or non-HA-enabled client proxy as appropriate.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.NameNodeProxies -->
+ <!-- start class org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo -->
+ <class name="NameNodeProxies.ProxyAndInfo" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ProxyAndInfo" type="PROXYTYPE, org.apache.hadoop.io.Text, java.net.InetSocketAddress"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getProxy" return="PROXYTYPE"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDelegationTokenService" return="org.apache.hadoop.io.Text"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getAddress" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Wrapper for a client proxy as well as its associated service ID.
+ This is simply used as a tuple-like return type for
+ {@link NameNodeProxies#createProxy} and
+ {@link NameNodeProxies#createNonHAProxy}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo -->
+ <!-- start interface org.apache.hadoop.hdfs.RemotePeerFactory -->
+ <interface name="RemotePeerFactory" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="newConnectedPeer" return="org.apache.hadoop.hdfs.net.Peer"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <param name="blockToken" type="org.apache.hadoop.security.token.Token"/>
+ <param name="datanodeId" type="org.apache.hadoop.hdfs.protocol.DatanodeID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[@param addr The address to connect to.
+ @param blockToken Token used during optional SASL negotiation
+ @param datanodeId ID of destination DataNode
+ @return A new Peer connected to the address.
+
+ @throws IOException If there was an error connecting or creating
+ the remote socket, encrypted stream, etc.]]>
+ </doc>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.hdfs.RemotePeerFactory -->
+ <!-- start class org.apache.hadoop.hdfs.UnknownCipherSuiteException -->
+ <class name="UnknownCipherSuiteException" extends="java.io.IOException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="UnknownCipherSuiteException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <doc>
+ <![CDATA[Thrown when an unknown cipher suite is encountered.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.UnknownCipherSuiteException -->
+ <!-- start class org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException -->
+ <class name="UnknownCryptoProtocolVersionException" extends="java.io.IOException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="UnknownCryptoProtocolVersionException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="UnknownCryptoProtocolVersionException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException -->
+ <doc>
+ <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}. This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files
+have strictly one writer at any one time. Bytes are always appended
+to the end of the writer's stream. There is no notion of "record appends"
+or "mutations" that are then checked or reordered. Writers simply emit
+a byte stream. That byte stream is guaranteed to be stored in the
+order written.</p>]]>
+ </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.client">
+ <!-- start class org.apache.hadoop.hdfs.client.BlockReportOptions -->
+ <class name="BlockReportOptions" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="isIncremental" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Options that can be specified when manually triggering a block report.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.client.BlockReportOptions -->
+ <!-- start class org.apache.hadoop.hdfs.client.BlockReportOptions.Factory -->
+ <class name="BlockReportOptions.Factory" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Factory"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setIncremental" return="org.apache.hadoop.hdfs.client.BlockReportOptions.Factory"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="incremental" type="boolean"/>
+ </method>
+ <method name="build" return="org.apache.hadoop.hdfs.client.BlockReportOptions"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.client.BlockReportOptions.Factory -->
+ <!-- start class org.apache.hadoop.hdfs.client.HdfsAdmin -->
+ <class name="HdfsAdmin" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HdfsAdmin" type="java.net.URI, org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Create a new HdfsAdmin client.
+
+ @param uri the unique URI of the HDFS file system to administer
+ @param conf configuration
+ @throws IOException in the event the file system could not be created]]>
+ </doc>
+ </constructor>
+ <method name="setQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="quota" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the namespace quota (count of files, directories, and sym links) for a
+ directory.
+
+ @param src the path to set the quota for
+ @param quota the value to set for the quota
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="clearQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Clear the namespace quota (count of files, directories and sym links) for a
+ directory.
+
+ @param src the path to clear the quota of
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="setSpaceQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="spaceQuota" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the storage space quota (size of files) for a directory. Note that
+ directories and sym links do not occupy storage space.
+
+ @param src the path to set the space quota of
+ @param spaceQuota the value to set for the space quota
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="clearSpaceQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Clear the storage space quota (size of files) for a directory. Note that
+ directories and sym links do not occupy storage space.
+
+ @param src the path to clear the space quota of
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="setQuotaByStorageType"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+ <param name="quota" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the quota by storage type for a directory. Note that
+ directories and sym links do not occupy storage type quota.
+
+ @param src the target directory to set the quota by storage type
+ @param type the storage type to set for quota by storage type
+ @param quota the value to set for quota by storage type
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="clearQuotaByStorageType"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Clear the space quota by storage type for a directory. Note that
+ directories and sym links do not occupy storage type quota.
+
+ @param src the target directory to clear the quota by storage type
+ @param type the storage type to clear for quota by storage type
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="allowSnapshot"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Allow snapshot on a directory.
+ @param path The path of the directory where snapshots will be taken.]]>
+ </doc>
+ </method>
+ <method name="disallowSnapshot"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Disallow snapshot on a directory.
+ @param path The path of the snapshottable directory.]]>
+ </doc>
+ </method>
+ <method name="addCacheDirective" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo"/>
+ <param name="flags" type="java.util.EnumSet"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Add a new CacheDirectiveInfo.
+
+ @param info Information about a directive to add.
+ @param flags {@link CacheFlag}s to use for this operation.
+ @return the ID of the directive that was created.
+ @throws IOException if the directive could not be added]]>
+ </doc>
+ </method>
+ <method name="modifyCacheDirective"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo"/>
+ <param name="flags" type="java.util.EnumSet"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Modify a CacheDirective.
+
+ @param info Information about the directive to modify. You must set the ID
+ to indicate which CacheDirective you want to modify.
+ @param flags {@link CacheFlag}s to use for this operation.
+ @throws IOException if the directive could not be modified]]>
+ </doc>
+ </method>
+ <method name="removeCacheDirective"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Remove a CacheDirective.
+
+ @param id identifier of the CacheDirectiveInfo to remove
+ @throws IOException if the directive could not be removed]]>
+ </doc>
+ </method>
+ <method name="listCacheDirectives" return="org.apache.hadoop.fs.RemoteIterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="filter" type="org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[List cache directives. Incrementally fetches results from the server.
+
+ @param filter Filter parameters to use when listing the directives, null to
+ list all directives visible to us.
+ @return A RemoteIterator which returns CacheDirectiveInfo objects.]]>
+ </doc>
+ </method>
+ <method name="addCachePool"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CachePoolInfo"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Add a cache pool.
+
+ @param info
+ The request to add a cache pool.
+ @throws IOException
+ If the request could not be completed.]]>
+ </doc>
+ </method>
+ <method name="modifyCachePool"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CachePoolInfo"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Modify an existing cache pool.
+
+ @param info
+ The request to modify a cache pool.
+ @throws IOException
+ If the request could not be completed.]]>
+ </doc>
+ </method>
+ <method name="removeCachePool"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="poolName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Remove a cache pool.
+
+ @param poolName
+ Name of the cache pool to remove.
+ @throws IOException
+ if the cache pool did not exist, or could not be removed.]]>
+ </doc>
+ </method>
+ <method name="listCachePools" return="org.apache.hadoop.fs.RemoteIterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[List all cache pools.
+
+ @return A remote iterator from which you can get CachePoolEntry objects.
+ Requests will be made as needed.
+ @throws IOException
+ If there was an error listing cache pools.]]>
+ </doc>
+ </method>
+ <method name="createEncryptionZone"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <param name="keyName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+ <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+ <doc>
+ <![CDATA[Create an encryption zone rooted at an empty existing directory, using the
+ specified encryption key. An encryption zone has an associated encryption
+ key used when reading and writing files within the zone.
+
+ @param path The path of the root of the encryption zone. Must refer to
+ an empty, existing directory.
+ @param keyName Name of key available at the KeyProvider.
+ @throws IOException if there was a general IO exception
+ @throws AccessControlException if the caller does not have access to path
+ @throws FileNotFoundException if the path does not exist]]>
+ </doc>
+ </method>
+ <method name="getEncryptionZoneForPath" return="org.apache.hadoop.hdfs.protocol.EncryptionZone"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+ <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+ <doc>
+ <![CDATA[Get the path of the encryption zone for a given file or directory.
+
+ @param path The path to get the ez for.
+
+ @return The EncryptionZone of the ez, or null if path is not in an ez.
+ @throws IOException if there was a general IO exception
+ @throws AccessControlException if the caller does not have access to path
+ @throws FileNotFoundException if the path does not exist]]>
+ </doc>
+ </method>
+ <method name="listEncryptionZones" return="org.apache.hadoop.fs.RemoteIterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns a RemoteIterator which can be used to list the encryption zones
+ in HDFS. For large numbers of encryption zones, the iterator will fetch
+ the list of zones in a number of small batches.
+ <p/>
+ Since the list is fetched in batches, it does not represent a
+ consistent snapshot of the entire list of encryption zones.
+ <p/>
+ This method can only be called by HDFS superusers.]]>
+ </doc>
+ </method>
+ <method name="getInotifyEventStream" return="org.apache.hadoop.hdfs.DFSInotifyEventInputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Exposes a stream of namesystem events. Only events occurring after the
+ stream is created are available.
+ See {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream}
+ for information on stream usage.
+ See {@link org.apache.hadoop.hdfs.inotify.Event}
+ for information on the available events.
+ <p/>
+ Inotify users may want to tune the following HDFS parameters to
+ ensure that enough extra HDFS edits are saved to support inotify clients
+ that fall behind the current state of the namespace while reading events.
+ The default parameter values should generally be reasonable. If edits are
+ deleted before their corresponding events can be read, clients will see a
+ {@link org.apache.hadoop.hdfs.inotify.MissingEventsException} on
+ {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream} method calls.
+
+ It should generally be sufficient to tune these parameters:
+ dfs.namenode.num.extra.edits.retained
+ dfs.namenode.max.extra.edits.segments.retained
+
+ Parameters that affect the number of created segments and the number of
+ edits that are considered necessary, i.e. do not count towards the
+ dfs.namenode.num.extra.edits.retained quota):
+ dfs.namenode.checkpoint.period
+ dfs.namenode.checkpoint.txns
+ dfs.namenode.num.checkpoints.retained
+ dfs.ha.log-roll.period
+ <p/>
+ It is recommended that local journaling be configured
+ (dfs.namenode.edits.dir) for inotify (in addition to a shared journal)
+ so that edit transfers from the shared journal can be avoided.
+
+ @throws IOException If there was an error obtaining the stream.]]>
+ </doc>
+ </method>
+ <method name="getInotifyEventStream" return="org.apache.hadoop.hdfs.DFSInotifyEventInputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="lastReadTxid" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[A version of {@link HdfsAdmin#getInotifyEventStream()} meant for advanced
+ users who are aware of HDFS edits up to lastReadTxid (e.g. because they
+ have access to an FSImage inclusive of lastReadTxid) and only want to read
+ events after this point.]]>
+ </doc>
+ </method>
+ <method name="setStoragePolicy"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="policyName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the source path to the specified storage policy.
+
+ @param src The source path referring to either a directory or a file.
+ @param policyName The name of the storage policy.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[The public API for performing administrative functions on HDFS. Those writing
+ applications against HDFS should prefer this interface to directly accessing
+ functionality in DistributedFileSystem or DFSClient.
+
+ Note that this is distinct from the similarly-named {@link DFSAdmin}, which
+ is a class that provides the functionality for the CLI `hdfs dfsadmin ...'
+ commands.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.client.HdfsAdmin -->
+ <!-- start class org.apache.hadoop.hdfs.client.HdfsDataInputStream -->
+ <class name="HdfsDataInputStream" extends="org.apache.hadoop.fs.FSDataInputStream"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HdfsDataInputStream" type="org.apache.hadoop.hdfs.DFSInputStream"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <constructor name="HdfsDataInputStream" type="org.apache.hadoop.crypto.CryptoInputStream"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getWrappedStream" return="java.io.InputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get a reference to the wrapped output stream. We always want to return the
+ actual underlying InputStream, even when we're using a CryptoStream. e.g.
+ in the delegated methods below.
+
+ @return the underlying output stream]]>
+ </doc>
+ </method>
+ <method name="getCurrentDatanode" return="org.apache.hadoop.hdfs.protocol.DatanodeInfo"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the datanode from which the stream is currently reading.]]>
+ </doc>
+ </method>
+ <method name="getCurrentBlock" return="org.apache.hadoop.hdfs.protocol.ExtendedBlock"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the block containing the target position.]]>
+ </doc>
+ </method>
+ <method name="getAllBlocks" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the collection of blocks that has already been located.]]>
+ </doc>
+ </method>
+ <method name="getVisibleLength" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the visible length of the file. It will
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: MAPREDUCE-6763. Shuffle server
listen queue is too small (Jason Lowe via Varun Saxena)
Posted by cd...@apache.org.
MAPREDUCE-6763. Shuffle server listen queue is too small (Jason Lowe via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/723facfa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/723facfa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/723facfa
Branch: refs/heads/HDFS-9806
Commit: 723facfa408f469891e67084bdafa18841a0cf64
Parents: 763f049
Author: Varun Saxena <va...@apache.org>
Authored: Sat Aug 20 03:58:01 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Sat Aug 20 03:58:16 2016 +0530
----------------------------------------------------------------------
.../src/main/resources/mapred-default.xml | 6 ++++++
.../src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java | 6 ++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/723facfa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 33eece3..73aaa7a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -211,6 +211,12 @@
</property>
<property>
+ <name>mapreduce.shuffle.listen.queue.size</name>
+ <value>128</value>
+ <description>The length of the shuffle server listen queue.</description>
+</property>
+
+<property>
<name>mapreduce.shuffle.connection-keep-alive.enable</name>
<value>false</value>
<description>set to true to support keep-alive connections.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/723facfa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 8721ef5..7818c81 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -195,6 +195,10 @@ public class ShuffleHandler extends AuxiliaryService {
public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port";
public static final int DEFAULT_SHUFFLE_PORT = 13562;
+ public static final String SHUFFLE_LISTEN_QUEUE_SIZE =
+ "mapreduce.shuffle.listen.queue.size";
+ public static final int DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE = 128;
+
public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED =
"mapreduce.shuffle.connection-keep-alive.enable";
public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false;
@@ -504,6 +508,8 @@ public class ShuffleHandler extends AuxiliaryService {
} catch (Exception ex) {
throw new RuntimeException(ex);
}
+ bootstrap.setOption("backlog", conf.getInt(SHUFFLE_LISTEN_QUEUE_SIZE,
+ DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE));
bootstrap.setOption("child.keepAlive", true);
bootstrap.setPipelineFactory(pipelineFact);
port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDFS-9745.
TestSecureNNWithQJM#testSecureMode sometimes fails with timeouts. Contributed
by Xiao Chen
Posted by cd...@apache.org.
HDFS-9745. TestSecureNNWithQJM#testSecureMode sometimes fails with timeouts. Contributed by Xiao Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/126d165e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/126d165e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/126d165e
Branch: refs/heads/HDFS-9806
Commit: 126d165efd80e266a8309241f3cf059e358f5019
Parents: 6f9c346
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Aug 23 14:46:58 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Aug 23 14:46:58 2016 +0000
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/126d165e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java
index 9abfb9c..46b016f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java
@@ -79,7 +79,7 @@ public class TestSecureNNWithQJM {
private MiniJournalCluster mjc;
@Rule
- public Timeout timeout = new Timeout(30000);
+ public Timeout timeout = new Timeout(180000);
@BeforeClass
public static void init() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
deleted file mode 100644
index 5ba1871..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.IOException;
-import java.net.URI;
-import java.nio.file.AccessDeniedException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.Timeout;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSCredentialsProviderChain;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
-import static org.junit.Assert.*;
-
-/**
- * Tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic.
- *
- */
-public class TestS3AAWSCredentialsProvider {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class);
-
- @Rule
- public Timeout testTimeout = new Timeout(1 * 60 * 1000);
-
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- /**
- * Declare what exception to raise, and the text which must be found
- * in it.
- * @param exceptionClass class of exception
- * @param text text in exception
- */
- private void expectException(Class<? extends Throwable> exceptionClass,
- String text) {
- exception.expect(exceptionClass);
- exception.expectMessage(text);
- }
-
- @Test
- public void testBadConfiguration() throws IOException {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER, "no.such.class");
- try {
- createFailingFS(conf);
- } catch (IOException e) {
- if (!(e.getCause() instanceof ClassNotFoundException)) {
- LOG.error("Unexpected nested cause: {} in {}", e.getCause(), e, e);
- throw e;
- }
- }
- }
-
- /**
- * Create a filesystem, expect it to fail by raising an IOException.
- * Raises an assertion exception if in fact the FS does get instantiated.
- * @param conf configuration
- * @throws IOException an expected exception.
- */
- private void createFailingFS(Configuration conf) throws IOException {
- S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf);
- fs.listStatus(new Path("/"));
- fail("Expected exception - got " + fs);
- }
-
- static class BadCredentialsProvider implements AWSCredentialsProvider {
-
- @SuppressWarnings("unused")
- public BadCredentialsProvider(URI name, Configuration conf) {
- }
-
- @Override
- public AWSCredentials getCredentials() {
- return new BasicAWSCredentials("bad_key", "bad_secret");
- }
-
- @Override
- public void refresh() {
- }
- }
-
- @Test
- public void testBadCredentials() throws Exception {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER, BadCredentialsProvider.class.getName());
- try {
- createFailingFS(conf);
- } catch (AccessDeniedException e) {
- // expected
- }
- }
-
- static class GoodCredentialsProvider extends AWSCredentialsProviderChain {
-
- @SuppressWarnings("unused")
- public GoodCredentialsProvider(URI name, Configuration conf) {
- super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
- conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider());
- }
- }
-
- @Test
- public void testGoodProvider() throws Exception {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName());
- S3ATestUtils.createTestFileSystem(conf);
- }
-
- @Test
- public void testAnonymousProvider() throws Exception {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER,
- AnonymousAWSCredentialsProvider.class.getName());
- Path testFile = new Path(
- conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
- S3ATestUtils.useCSVDataEndpoint(conf);
- FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
- assertNotNull(fs);
- assertTrue(fs instanceof S3AFileSystem);
- FileStatus stat = fs.getFileStatus(testFile);
- assertNotNull(stat);
- assertEquals(testFile, stat.getPath());
- }
-
- /**
- * A credential provider whose constructor signature doesn't match.
- */
- static class ConstructorSignatureErrorProvider
- implements AWSCredentialsProvider {
-
- @SuppressWarnings("unused")
- public ConstructorSignatureErrorProvider(String str) {
- }
-
- @Override
- public AWSCredentials getCredentials() {
- return null;
- }
-
- @Override
- public void refresh() {
- }
- }
-
- /**
- * A credential provider whose constructor raises an NPE.
- */
- static class ConstructorFailureProvider
- implements AWSCredentialsProvider {
-
- @SuppressWarnings("unused")
- public ConstructorFailureProvider() {
- throw new NullPointerException("oops");
- }
-
- @Override
- public AWSCredentials getCredentials() {
- return null;
- }
-
- @Override
- public void refresh() {
- }
- }
-
- @Test
- public void testProviderWrongClass() throws Exception {
- expectProviderInstantiationFailure(this.getClass().getName(),
- NOT_AWS_PROVIDER);
- }
-
- @Test
- public void testProviderNotAClass() throws Exception {
- expectProviderInstantiationFailure("NoSuchClass",
- "ClassNotFoundException");
- }
-
- private void expectProviderInstantiationFailure(String option,
- String expectedErrorText) throws IOException {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER, option);
- Path testFile = new Path(
- conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
- expectException(IOException.class, expectedErrorText);
- URI uri = testFile.toUri();
- S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
- }
-
- @Test
- public void testProviderConstructorError() throws Exception {
- expectProviderInstantiationFailure(
- ConstructorSignatureErrorProvider.class.getName(),
- CONSTRUCTOR_EXCEPTION);
- }
-
- @Test
- public void testProviderFailureError() throws Exception {
- expectProviderInstantiationFailure(
- ConstructorFailureProvider.class.getName(),
- INSTANTIATION_EXCEPTION);
- }
-
- @Test
- public void testInstantiationChain() throws Throwable {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER,
- TemporaryAWSCredentialsProvider.NAME
- + ", \t" + SimpleAWSCredentialsProvider.NAME
- + " ,\n " + AnonymousAWSCredentialsProvider.NAME);
- Path testFile = new Path(
- conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
-
- URI uri = testFile.toUri();
- S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockingThreadPool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockingThreadPool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockingThreadPool.java
deleted file mode 100644
index bd738b2..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockingThreadPool.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-/**
- * Demonstrate that the threadpool blocks additional client requests if
- * its queue is full (rather than throwing an exception) by initiating an
- * upload consisting of 4 parts with 2 threads and 1 spot in the queue. The
- * 4th part should not trigger an exception as it would with a
- * non-blocking threadpool.
- */
-public class TestS3ABlockingThreadPool {
-
- private Configuration conf;
- private S3AFileSystem fs;
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- protected Path getTestPath() {
- return new Path("/tests3a");
- }
-
- @Before
- public void setUp() throws Exception {
- conf = new Configuration();
- conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024);
- conf.setLong(Constants.MULTIPART_SIZE, 5 * 1024 * 1024);
- conf.setInt(Constants.MAX_THREADS, 2);
- conf.setInt(Constants.MAX_TOTAL_TASKS, 1);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(getTestPath(), true);
- }
- }
-
- @Test
- public void testRegularMultiPartUpload() throws Exception {
- fs = S3ATestUtils.createTestFileSystem(conf);
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 *
- 1024);
- }
-
- @Test
- public void testFastMultiPartUpload() throws Exception {
- conf.setBoolean(Constants.FAST_UPLOAD, true);
- fs = S3ATestUtils.createTestFileSystem(conf);
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 *
- 1024);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlocksize.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlocksize.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlocksize.java
deleted file mode 100644
index 76fbf99..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlocksize.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.fileStatsToString;
-
-public class TestS3ABlocksize extends AbstractFSContractTestBase {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3ABlocksize.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Test
- @SuppressWarnings("deprecation")
- public void testBlockSize() throws Exception {
- FileSystem fs = getFileSystem();
- long defaultBlockSize = fs.getDefaultBlockSize();
- assertEquals("incorrect blocksize",
- S3AFileSystem.DEFAULT_BLOCKSIZE, defaultBlockSize);
- long newBlockSize = defaultBlockSize * 2;
- fs.getConf().setLong(Constants.FS_S3A_BLOCK_SIZE, newBlockSize);
-
- Path dir = path("testBlockSize");
- Path file = new Path(dir, "file");
- createFile(fs, file, true, dataset(1024, 'a', 'z' - 'a'));
- FileStatus fileStatus = fs.getFileStatus(file);
- assertEquals("Double default block size in stat(): " + fileStatus,
- newBlockSize,
- fileStatus.getBlockSize());
-
- // check the listing & assert that the block size is picked up by
- // this route too.
- boolean found = false;
- FileStatus[] listing = fs.listStatus(dir);
- for (FileStatus stat : listing) {
- LOG.info("entry: {}", stat);
- if (file.equals(stat.getPath())) {
- found = true;
- assertEquals("Double default block size in ls(): " + stat,
- newBlockSize,
- stat.getBlockSize());
- }
- }
- assertTrue("Did not find " + fileStatsToString(listing, ", "), found);
- }
-
- @Test
- public void testRootFileStatusHasBlocksize() throws Throwable {
- FileSystem fs = getFileSystem();
- FileStatus status = fs.getFileStatus(new Path("/"));
- assertTrue("Invalid root blocksize",
- status.getBlockSize() >= 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
deleted file mode 100644
index 29bfd59..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
+++ /dev/null
@@ -1,429 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.reflect.FieldUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3native.S3xLoginHelper;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.net.URI;
-
-import org.apache.hadoop.security.ProviderUtils;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.http.HttpStatus;
-import org.junit.rules.TemporaryFolder;
-
-public class TestS3AConfiguration {
- private static final String EXAMPLE_ID = "AKASOMEACCESSKEY";
- private static final String EXAMPLE_KEY =
- "RGV0cm9pdCBSZ/WQgY2xl/YW5lZCB1cAEXAMPLE";
-
- private Configuration conf;
- private S3AFileSystem fs;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AConfiguration.class);
-
- private static final String TEST_ENDPOINT = "test.fs.s3a.endpoint";
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Rule
- public final TemporaryFolder tempDir = new TemporaryFolder();
-
- /**
- * Test if custom endpoint is picked up.
- * <p/>
- * The test expects TEST_ENDPOINT to be defined in the Configuration
- * describing the endpoint of the bucket to which TEST_FS_S3A_NAME points
- * (f.i. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland).
- * Evidently, the bucket has to be hosted in the region denoted by the
- * endpoint for the test to succeed.
- * <p/>
- * More info and the list of endpoint identifiers:
- * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
- *
- * @throws Exception
- */
- @Test
- public void testEndpoint() throws Exception {
- conf = new Configuration();
- String endpoint = conf.getTrimmed(TEST_ENDPOINT, "");
- if (endpoint.isEmpty()) {
- LOG.warn("Custom endpoint test skipped as " + TEST_ENDPOINT + "config " +
- "setting was not detected");
- } else {
- conf.set(Constants.ENDPOINT, endpoint);
- fs = S3ATestUtils.createTestFileSystem(conf);
- AmazonS3Client s3 = fs.getAmazonS3Client();
- String endPointRegion = "";
- // Differentiate handling of "s3-" and "s3." based endpoint identifiers
- String[] endpointParts = StringUtils.split(endpoint, '.');
- if (endpointParts.length == 3) {
- endPointRegion = endpointParts[0].substring(3);
- } else if (endpointParts.length == 4) {
- endPointRegion = endpointParts[1];
- } else {
- fail("Unexpected endpoint");
- }
- assertEquals("Endpoint config setting and bucket location differ: ",
- endPointRegion, s3.getBucketLocation(fs.getUri().getHost()));
- }
- }
-
- @Test
- public void testProxyConnection() throws Exception {
- conf = new Configuration();
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.set(Constants.PROXY_HOST, "127.0.0.1");
- conf.setInt(Constants.PROXY_PORT, 1);
- String proxy =
- conf.get(Constants.PROXY_HOST) + ":" + conf.get(Constants.PROXY_PORT);
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server at " + proxy);
- } catch (AWSClientIOException e) {
- // expected
- }
- }
-
- @Test
- public void testProxyPortWithoutHost() throws Exception {
- conf = new Configuration();
- conf.unset(Constants.PROXY_HOST);
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.setInt(Constants.PROXY_PORT, 1);
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a proxy configuration error");
- } catch (IllegalArgumentException e) {
- String msg = e.toString();
- if (!msg.contains(Constants.PROXY_HOST) &&
- !msg.contains(Constants.PROXY_PORT)) {
- throw e;
- }
- }
- }
-
- @Test
- public void testAutomaticProxyPortSelection() throws Exception {
- conf = new Configuration();
- conf.unset(Constants.PROXY_PORT);
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.set(Constants.PROXY_HOST, "127.0.0.1");
- conf.set(Constants.SECURE_CONNECTIONS, "true");
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server");
- } catch (AWSClientIOException e) {
- // expected
- }
- conf.set(Constants.SECURE_CONNECTIONS, "false");
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server");
- } catch (AWSClientIOException e) {
- // expected
- }
- }
-
- @Test
- public void testUsernameInconsistentWithPassword() throws Exception {
- conf = new Configuration();
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.set(Constants.PROXY_HOST, "127.0.0.1");
- conf.setInt(Constants.PROXY_PORT, 1);
- conf.set(Constants.PROXY_USERNAME, "user");
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server");
- } catch (IllegalArgumentException e) {
- String msg = e.toString();
- if (!msg.contains(Constants.PROXY_USERNAME) &&
- !msg.contains(Constants.PROXY_PASSWORD)) {
- throw e;
- }
- }
- conf = new Configuration();
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.set(Constants.PROXY_HOST, "127.0.0.1");
- conf.setInt(Constants.PROXY_PORT, 1);
- conf.set(Constants.PROXY_PASSWORD, "password");
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server");
- } catch (IllegalArgumentException e) {
- String msg = e.toString();
- if (!msg.contains(Constants.PROXY_USERNAME) &&
- !msg.contains(Constants.PROXY_PASSWORD)) {
- throw e;
- }
- }
- }
-
- @Test
- public void testCredsFromCredentialProvider() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- provisionAccessKeys(conf);
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
- }
-
- void provisionAccessKeys(final Configuration conf) throws Exception {
- // add our creds to the provider
- final CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
- provider.createCredentialEntry(Constants.ACCESS_KEY,
- EXAMPLE_ID.toCharArray());
- provider.createCredentialEntry(Constants.SECRET_KEY,
- EXAMPLE_KEY.toCharArray());
- provider.flush();
- }
-
- @Test
- public void testCredsFromUserInfo() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- provisionAccessKeys(conf);
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
- URI uriWithUserInfo = new URI("s3a://123:456@foobar");
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
- assertEquals("AccessKey incorrect.", "123", creds.getUser());
- assertEquals("SecretKey incorrect.", "456", creds.getPassword());
- }
-
- @Test
- public void testIDFromUserInfoSecretFromCredentialProvider()
- throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- provisionAccessKeys(conf);
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
- URI uriWithUserInfo = new URI("s3a://123@foobar");
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
- assertEquals("AccessKey incorrect.", "123", creds.getUser());
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
- }
-
- @Test
- public void testSecretFromCredentialProviderIDFromConfig() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- // add our creds to the provider
- final CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
- provider.createCredentialEntry(Constants.SECRET_KEY,
- EXAMPLE_KEY.toCharArray());
- provider.flush();
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID);
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
- }
-
- @Test
- public void testIDFromCredentialProviderSecretFromConfig() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- // add our creds to the provider
- final CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
- provider.createCredentialEntry(Constants.ACCESS_KEY,
- EXAMPLE_ID.toCharArray());
- provider.flush();
-
- conf.set(Constants.SECRET_KEY, EXAMPLE_KEY);
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
- }
-
- @Test
- public void testExcludingS3ACredentialProvider() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- "jceks://s3a/foobar," + jks.toString());
-
- // first make sure that the s3a based provider is removed
- Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
- conf, S3AFileSystem.class);
- String newPath = conf.get(
- CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
- assertFalse("Provider Path incorrect", newPath.contains("s3a://"));
-
- // now let's make sure the new path is created by the S3AFileSystem
- // and the integration still works. Let's provision the keys through
- // the altered configuration instance and then try and access them
- // using the original config with the s3a provider in the path.
- provisionAccessKeys(c);
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
- URI uriWithUserInfo = new URI("s3a://123:456@foobar");
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
- assertEquals("AccessKey incorrect.", "123", creds.getUser());
- assertEquals("SecretKey incorrect.", "456", creds.getPassword());
-
- }
-
- @Test
- public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() throws Exception {
-
- conf = new Configuration();
- conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true));
- assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false));
-
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
- assertNotNull(s3);
- S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class,
- "clientOptions");
- assertTrue("Expected to find path style access to be switched on!",
- clientOptions.isPathStyleAccess());
- byte[] file = ContractTestUtils.toAsciiByteArray("test file");
- ContractTestUtils.writeAndRead(fs, new Path("/path/style/access/testFile"), file, file.length, conf.getInt(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true);
- } catch (final AWSS3IOException e) {
- LOG.error("Caught exception: ", e);
- // Catch/pass standard path style access behaviour when live bucket
- // isn't in the same region as the s3 client default. See
- // http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
- assertEquals(e.getStatusCode(), HttpStatus.SC_MOVED_PERMANENTLY);
- }
- }
-
- @Test
- public void testDefaultUserAgent() throws Exception {
- conf = new Configuration();
- fs = S3ATestUtils.createTestFileSystem(conf);
- assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
- assertNotNull(s3);
- ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
- "clientConfiguration");
- assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent());
- }
-
- @Test
- public void testCustomUserAgent() throws Exception {
- conf = new Configuration();
- conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
- fs = S3ATestUtils.createTestFileSystem(conf);
- assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
- assertNotNull(s3);
- ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
- "clientConfiguration");
- assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
- awsConf.getUserAgent());
- }
-
- /**
- * Reads and returns a field from an object using reflection. If the field
- * cannot be found, is null, or is not the expected type, then this method
- * fails the test.
- *
- * @param target object to read
- * @param fieldType type of field to read, which will also be the return type
- * @param fieldName name of field to read
- * @return field that was read
- * @throws IllegalAccessException if access not allowed
- */
- private static <T> T getField(Object target, Class<T> fieldType,
- String fieldName) throws IllegalAccessException {
- Object obj = FieldUtils.readField(target, fieldName, true);
- assertNotNull(String.format(
- "Could not read field named %s in object with class %s.", fieldName,
- target.getClass().getName()), obj);
- assertTrue(String.format(
- "Unexpected type found for field named %s, expected %s, actual %s.",
- fieldName, fieldType.getName(), obj.getClass().getName()),
- fieldType.isAssignableFrom(obj.getClass()));
- return fieldType.cast(obj);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ACredentialsInURL.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ACredentialsInURL.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ACredentialsInURL.java
deleted file mode 100644
index 8cb7c0f..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ACredentialsInURL.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URLEncoder;
-import java.nio.file.AccessDeniedException;
-
-import static org.apache.hadoop.fs.s3a.S3ATestConstants.TEST_FS_S3A_NAME;
-
-/**
- * Tests that credentials can go into the URL. This includes a valid
- * set, and a check that an invalid set do at least get stripped out
- * of the final URI
- */
-public class TestS3ACredentialsInURL extends Assert {
- private S3AFileSystem fs;
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3ACredentialsInURL.class);
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @After
- public void teardown() {
- IOUtils.closeStream(fs);
- }
-
- /**
- * Test instantiation.
- * @throws Throwable
- */
- @Test
- public void testInstantiateFromURL() throws Throwable {
-
- Configuration conf = new Configuration();
- String accessKey = conf.get(Constants.ACCESS_KEY);
- String secretKey = conf.get(Constants.SECRET_KEY);
- String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, "");
- Assume.assumeNotNull(fsname, accessKey, secretKey);
- URI original = new URI(fsname);
- URI secretsURI = createUriWithEmbeddedSecrets(original,
- accessKey, secretKey);
- if (secretKey.contains("/")) {
- assertTrue("test URI encodes the / symbol", secretsURI.toString().
- contains("%252F"));
- }
- if (secretKey.contains("+")) {
- assertTrue("test URI encodes the + symbol", secretsURI.toString().
- contains("%252B"));
- }
- assertFalse("Does not contain secrets", original.equals(secretsURI));
-
- conf.set(TEST_FS_S3A_NAME, secretsURI.toString());
- conf.unset(Constants.ACCESS_KEY);
- conf.unset(Constants.SECRET_KEY);
- fs = S3ATestUtils.createTestFileSystem(conf);
- String fsURI = fs.getUri().toString();
- assertFalse("FS URI contains a @ symbol", fsURI.contains("@"));
- assertFalse("FS URI contains a % symbol", fsURI.contains("%"));
- if (!original.toString().startsWith(fsURI)) {
- fail("Filesystem URI does not match original");
- }
- validate("original path", new Path(original));
- validate("bare path", new Path("/"));
- validate("secrets path", new Path(secretsURI));
- }
-
- private void validate(String text, Path path) throws IOException {
- try {
- fs.canonicalizeUri(path.toUri());
- fs.checkPath(path);
- assertTrue(text + " Not a directory",
- fs.getFileStatus(new Path("/")).isDirectory());
- fs.globStatus(path);
- } catch (AssertionError e) {
- throw e;
- } catch (Exception e) {
- LOG.debug("{} failure: {}", text, e, e);
- fail(text + " Test failed");
- }
- }
-
- /**
- * Set up some invalid credentials, verify login is rejected.
- * @throws Throwable
- */
- @Test
- public void testInvalidCredentialsFail() throws Throwable {
- Configuration conf = new Configuration();
- String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, "");
- Assume.assumeNotNull(fsname);
- URI original = new URI(fsname);
- URI testURI = createUriWithEmbeddedSecrets(original, "user", "//");
-
- conf.set(TEST_FS_S3A_NAME, testURI.toString());
- fs = S3ATestUtils.createTestFileSystem(conf);
- try {
- S3AFileStatus status = fs.getFileStatus(new Path("/"));
- fail("Expected an AccessDeniedException, got " + status);
- } catch (AccessDeniedException e) {
- // expected
- }
-
- }
-
- private URI createUriWithEmbeddedSecrets(URI original,
- String accessKey,
- String secretKey) throws UnsupportedEncodingException {
- String encodedSecretKey = URLEncoder.encode(secretKey, "UTF-8");
- String formattedString = String.format("%s://%s:%s@%s/%s/",
- original.getScheme(),
- accessKey,
- encodedSecretKey,
- original.getHost(),
- original.getPath());
- URI testURI;
- try {
- testURI = new Path(formattedString).toUri();
- } catch (IllegalArgumentException e) {
- // inner cause is stripped to keep any secrets out of stack traces
- throw new IllegalArgumentException("Could not encode Path");
- }
- return testURI;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryption.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryption.java
deleted file mode 100644
index 43a26ff..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryption.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-
-/**
- * Test whether or not encryption works by turning it on. Some checks
- * are made for different file sizes as there have been reports that the
- * file length may be rounded up to match word boundaries.
- */
-public class TestS3AEncryption extends AbstractS3ATestBase {
- private static final String AES256 = Constants.SERVER_SIDE_ENCRYPTION_AES256;
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- S3ATestUtils.disableFilesystemCaching(conf);
- conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
- AES256);
- return conf;
- }
-
- private static final int[] SIZES = {
- 0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 10 - 3, 2 ^ 11 - 2, 2 ^ 12 - 1
- };
-
- @Override
- public void teardown() throws Exception {
- super.teardown();
- IOUtils.closeStream(getFileSystem());
- }
-
- @Test
- public void testEncryption() throws Throwable {
- for (int size: SIZES) {
- validateEncryptionForFilesize(size);
- }
- }
-
- @Test
- public void testEncryptionOverRename() throws Throwable {
- skipIfEncryptionTestsDisabled(getConfiguration());
- Path src = path(createFilename(1024));
- byte[] data = dataset(1024, 'a', 'z');
- S3AFileSystem fs = getFileSystem();
- writeDataset(fs, src, data, data.length, 1024 * 1024, true);
- ContractTestUtils.verifyFileContents(fs, src, data);
- Path dest = path(src.getName() + "-copy");
- fs.rename(src, dest);
- ContractTestUtils.verifyFileContents(fs, dest, data);
- assertEncrypted(dest);
- }
-
- protected void validateEncryptionForFilesize(int len) throws IOException {
- skipIfEncryptionTestsDisabled(getConfiguration());
- describe("Create an encrypted file of size " + len);
- String src = createFilename(len);
- Path path = writeThenReadFile(src, len);
- assertEncrypted(path);
- rm(getFileSystem(), path, false, false);
- }
-
- private String createFilename(int len) {
- return String.format("%s-%04x", methodName.getMethodName(), len);
- }
-
- /**
- * Assert that at path references an encrypted blob.
- * @param path path
- * @throws IOException on a failure
- */
- private void assertEncrypted(Path path) throws IOException {
- ObjectMetadata md = getFileSystem().getObjectMetadata(path);
- assertEquals(AES256, md.getSSEAlgorithm());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionAlgorithmPropagation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionAlgorithmPropagation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionAlgorithmPropagation.java
deleted file mode 100644
index 04ca830..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionAlgorithmPropagation.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-
-/**
- * Test whether or not encryption settings propagate by choosing an invalid
- * one. We expect the write to fail with a 400 bad request error
- */
-public class TestS3AEncryptionAlgorithmPropagation extends AbstractS3ATestBase {
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- S3ATestUtils.disableFilesystemCaching(conf);
- conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
- "DES");
- return conf;
- }
-
- @Override
- public void teardown() throws Exception {
- super.teardown();
- IOUtils.closeStream(getFileSystem());
- }
-
- @Test
- public void testEncrypt0() throws Throwable {
- writeThenReadFileToFailure(0);
- }
-
- @Test
- public void testEncrypt256() throws Throwable {
- writeThenReadFileToFailure(256);
- }
-
- /**
- * Make this a no-op so test setup doesn't fail.
- * @param path path path
- * @throws IOException on any failure
- */
- @Override
- protected void mkdirs(Path path) throws IOException {
-
- }
-
- protected void writeThenReadFileToFailure(int len) throws IOException {
- skipIfEncryptionTestsDisabled(getConfiguration());
- describe("Create an encrypted file of size " + len);
- try {
- writeThenReadFile(methodName.getMethodName() + '-' + len, len);
- fail("Expected an exception about an illegal encryption algorithm");
- } catch (AWSS3IOException e) {
- assertStatusCode(e, 400);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionFastOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionFastOutputStream.java
deleted file mode 100644
index 1fa8486..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionFastOutputStream.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Run the encryption tests against the Fast output stream.
- * This verifies that both file writing paths can encrypt their data.
- */
-public class TestS3AEncryptionFastOutputStream extends TestS3AEncryption {
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- conf.setBoolean(Constants.FAST_UPLOAD, true);
- return conf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java
deleted file mode 100644
index 58ac496..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.InvalidRequestException;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.nio.file.AccessDeniedException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
-
-/**
- * Test S3A Failure translation, including a functional test
- * generating errors during stream IO.
- */
-public class TestS3AFailureHandling extends AbstractFSContractTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AFailureHandling.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Test
- public void testReadFileChanged() throws Throwable {
- describe("overwrite a file with a shorter one during a read, seek");
- final int fullLength = 8192;
- final byte[] fullDataset = dataset(fullLength, 'a', 32);
- final int shortLen = 4096;
- final byte[] shortDataset = dataset(shortLen, 'A', 32);
- final FileSystem fs = getFileSystem();
- final Path testpath = path("readFileToChange.txt");
- // initial write
- writeDataset(fs, testpath, fullDataset, fullDataset.length, 1024, false);
- try(FSDataInputStream instream = fs.open(testpath)) {
- instream.seek(fullLength - 16);
- assertTrue("no data to read", instream.read() >= 0);
- // overwrite
- writeDataset(fs, testpath, shortDataset, shortDataset.length, 1024, true);
- // here the file length is less. Probe the file to see if this is true,
- // with a spin and wait
- eventually(30 *1000, new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- assertEquals(shortLen, fs.getFileStatus(testpath).getLen());
- return null;
- }
- });
- // here length is shorter. Assuming it has propagated to all replicas,
- // the position of the input stream is now beyond the EOF.
- // An attempt to seek backwards to a position greater than the
- // short length will raise an exception from AWS S3, which must be
- // translated into an EOF
-
- instream.seek(shortLen + 1024);
- int c = instream.read();
- assertIsEOF("read()", c);
-
- byte[] buf = new byte[256];
-
- assertIsEOF("read(buffer)", instream.read(buf));
- assertIsEOF("read(offset)",
- instream.read(instream.getPos(), buf, 0, buf.length));
-
- // now do a block read fully, again, backwards from the current pos
- try {
- instream.readFully(shortLen + 512, buf);
- fail("Expected readFully to fail");
- } catch (EOFException expected) {
- LOG.debug("Expected EOF: ", expected);
- }
-
- assertIsEOF("read(offset)",
- instream.read(shortLen + 510, buf, 0, buf.length));
-
- // seek somewhere useful
- instream.seek(shortLen - 256);
-
- // delete the file. Reads must fail
- fs.delete(testpath, false);
-
- try {
- int r = instream.read();
- fail("Expected an exception, got " + r);
- } catch (FileNotFoundException e) {
- // expected
- }
-
- try {
- instream.readFully(2048, buf);
- fail("Expected readFully to fail");
- } catch (FileNotFoundException e) {
- // expected
- }
-
- }
- }
-
- /**
- * Assert that a read operation returned an EOF value.
- * @param operation specific operation
- * @param readResult result
- */
- private void assertIsEOF(String operation, int readResult) {
- assertEquals("Expected EOF from "+ operation
- + "; got char " + (char) readResult, -1, readResult);
- }
-
- @Test
- public void test404isNotFound() throws Throwable {
- verifyTranslated(FileNotFoundException.class, createS3Exception(404));
- }
-
- protected Exception verifyTranslated(Class clazz,
- AmazonClientException exception) throws Exception {
- return verifyExceptionClass(clazz,
- translateException("test", "/", exception));
- }
-
- @Test
- public void test401isNotPermittedFound() throws Throwable {
- verifyTranslated(AccessDeniedException.class,
- createS3Exception(401));
- }
-
- protected AmazonS3Exception createS3Exception(int code) {
- AmazonS3Exception source = new AmazonS3Exception("");
- source.setStatusCode(code);
- return source;
- }
-
- @Test
- public void testGenericS3Exception() throws Throwable {
- // S3 exception of no known type
- AWSS3IOException ex = (AWSS3IOException)verifyTranslated(
- AWSS3IOException.class,
- createS3Exception(451));
- assertEquals(451, ex.getStatusCode());
- }
-
- @Test
- public void testGenericServiceS3Exception() throws Throwable {
- // service exception of no known type
- AmazonServiceException ase = new AmazonServiceException("unwind");
- ase.setStatusCode(500);
- AWSServiceIOException ex = (AWSServiceIOException)verifyTranslated(
- AWSServiceIOException.class,
- ase);
- assertEquals(500, ex.getStatusCode());
- }
-
- @Test
- public void testGenericClientException() throws Throwable {
- // Generic Amazon exception
- verifyTranslated(AWSClientIOException.class,
- new AmazonClientException(""));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java
deleted file mode 100644
index e507cf6..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-/**
- * Tests regular and multi-part upload functionality for S3AFastOutputStream.
- * File sizes are kept small to reduce test duration on slow connections
- */
-public class TestS3AFastOutputStream {
- private FileSystem fs;
-
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024);
- conf.setInt(Constants.MULTIPART_SIZE, 5 * 1024 * 1024);
- conf.setBoolean(Constants.FAST_UPLOAD, true);
- fs = S3ATestUtils.createTestFileSystem(conf);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(getTestPath(), true);
- }
- }
-
- protected Path getTestPath() {
- return new Path("/tests3a");
- }
-
- @Test
- public void testRegularUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
- }
-
- @Test
- public void testMultiPartUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 *
- 1024);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java
deleted file mode 100644
index 0a8dd2d..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.net.URI;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.Statistic.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.MetricDiff;
-import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
-
-/**
- * Use metrics to assert about the cost of file status queries.
- * {@link S3AFileSystem#getFileStatus(Path)}.
- */
-public class TestS3AFileOperationCost extends AbstractFSContractTestBase {
-
- private MetricDiff metadataRequests;
- private MetricDiff listRequests;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AFileOperationCost.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Override
- public S3AFileSystem getFileSystem() {
- return (S3AFileSystem) super.getFileSystem();
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
- S3AFileSystem fs = getFileSystem();
- metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
- listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
- }
-
- @Test
- public void testCostOfGetFileStatusOnFile() throws Throwable {
- describe("performing getFileStatus on a file");
- Path simpleFile = path("simple.txt");
- S3AFileSystem fs = getFileSystem();
- touch(fs, simpleFile);
- resetMetricDiffs();
- S3AFileStatus status = fs.getFileStatus(simpleFile);
- assertTrue("not a file: " + status, status.isFile());
- metadataRequests.assertDiffEquals(1);
- listRequests.assertDiffEquals(0);
- }
-
- private void resetMetricDiffs() {
- reset(metadataRequests, listRequests);
- }
-
- @Test
- public void testCostOfGetFileStatusOnEmptyDir() throws Throwable {
- describe("performing getFileStatus on an empty directory");
- S3AFileSystem fs = getFileSystem();
- Path dir = path("empty");
- fs.mkdirs(dir);
- resetMetricDiffs();
- S3AFileStatus status = fs.getFileStatus(dir);
- assertTrue("not empty: " + status, status.isEmptyDirectory());
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(0);
- }
-
- @Test
- public void testCostOfGetFileStatusOnMissingFile() throws Throwable {
- describe("performing getFileStatus on a missing file");
- S3AFileSystem fs = getFileSystem();
- Path path = path("missing");
- resetMetricDiffs();
- try {
- S3AFileStatus status = fs.getFileStatus(path);
- fail("Got a status back from a missing file path " + status);
- } catch (FileNotFoundException expected) {
- // expected
- }
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(1);
- }
-
- @Test
- public void testCostOfGetFileStatusOnMissingSubPath() throws Throwable {
- describe("performing getFileStatus on a missing file");
- S3AFileSystem fs = getFileSystem();
- Path path = path("missingdir/missingpath");
- resetMetricDiffs();
- try {
- S3AFileStatus status = fs.getFileStatus(path);
- fail("Got a status back from a missing file path " + status);
- } catch (FileNotFoundException expected) {
- // expected
- }
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(1);
- }
-
- @Test
- public void testCostOfGetFileStatusOnNonEmptyDir() throws Throwable {
- describe("performing getFileStatus on a non-empty directory");
- S3AFileSystem fs = getFileSystem();
- Path dir = path("empty");
- fs.mkdirs(dir);
- Path simpleFile = new Path(dir, "simple.txt");
- touch(fs, simpleFile);
- resetMetricDiffs();
- S3AFileStatus status = fs.getFileStatus(dir);
- if (status.isEmptyDirectory()) {
- // erroneous state
- String fsState = fs.toString();
- fail("FileStatus says directory isempty: " + status
- + "\n" + ContractTestUtils.ls(fs, dir)
- + "\n" + fsState);
- }
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(1);
- }
-
- @Test
- public void testCostOfCopyFromLocalFile() throws Throwable {
- describe("testCostOfCopyFromLocalFile");
- File localTestDir = getTestDir("tmp");
- localTestDir.mkdirs();
- File tmpFile = File.createTempFile("tests3acost", ".txt",
- localTestDir);
- tmpFile.delete();
- try {
- URI localFileURI = tmpFile.toURI();
- FileSystem localFS = FileSystem.get(localFileURI,
- getFileSystem().getConf());
- Path localPath = new Path(localFileURI);
- int len = 10 * 1024;
- byte[] data = dataset(len, 'A', 'Z');
- writeDataset(localFS, localPath, data, len, 1024, true);
- S3AFileSystem s3a = getFileSystem();
- MetricDiff copyLocalOps = new MetricDiff(s3a,
- INVOCATION_COPY_FROM_LOCAL_FILE);
- MetricDiff putRequests = new MetricDiff(s3a,
- OBJECT_PUT_REQUESTS);
- MetricDiff putBytes = new MetricDiff(s3a,
- OBJECT_PUT_BYTES);
-
- Path remotePath = path("copied");
- s3a.copyFromLocalFile(false, true, localPath, remotePath);
- verifyFileContents(s3a, remotePath, data);
- copyLocalOps.assertDiffEquals(1);
- putRequests.assertDiffEquals(1);
- putBytes.assertDiffEquals(len);
- // print final stats
- LOG.info("Filesystem {}", s3a);
- } finally {
- tmpFile.delete();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java
deleted file mode 100644
index 6315a80..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystemContractBaseTest;
-import org.apache.hadoop.fs.Path;
-
-/**
- * Tests a live S3 system. If your keys and bucket aren't specified, all tests
- * are marked as passed.
- *
- * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
- * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
- * properly making it impossible to skip the tests if we don't have a valid
- * bucket.
- **/
-public class TestS3AFileSystemContract extends FileSystemContractBaseTest {
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestS3AFileSystemContract.class);
-
- @Override
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
-
- fs = S3ATestUtils.createTestFileSystem(conf);
- super.setUp();
- }
-
- @Override
- protected void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(path("test"), true);
- }
- super.tearDown();
- }
-
- @Override
- public void testMkdirsWithUmask() throws Exception {
- // not supported
- }
-
- @Override
- public void testRenameFileAsExistingFile() throws Exception {
- if (!renameSupported()) return;
-
- Path src = path("/test/hadoop/file");
- createFile(src);
- Path dst = path("/test/new/newfile");
- createFile(dst);
- // s3 doesn't support rename option
- // rename-overwrites-dest is always allowed.
- rename(src, dst, true, false, true);
- }
-
- @Override
- public void testRenameDirectoryAsExistingDirectory() throws Exception {
- if (!renameSupported()) {
- return;
- }
-
- Path src = path("/test/hadoop/dir");
- fs.mkdirs(src);
- createFile(path("/test/hadoop/dir/file1"));
- createFile(path("/test/hadoop/dir/subdir/file2"));
-
- Path dst = path("/test/new/newdir");
- fs.mkdirs(dst);
- rename(src, dst, true, false, true);
- assertFalse("Nested file1 exists",
- fs.exists(path("/test/hadoop/dir/file1")));
- assertFalse("Nested file2 exists",
- fs.exists(path("/test/hadoop/dir/subdir/file2")));
- assertTrue("Renamed nested file1 exists",
- fs.exists(path("/test/new/newdir/file1")));
- assertTrue("Renamed nested exists",
- fs.exists(path("/test/new/newdir/subdir/file2")));
- }
-
-// @Override
- public void testMoveDirUnderParent() throws Throwable {
- // not support because
- // Fails if dst is a directory that is not empty.
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
deleted file mode 100644
index 957dfb9..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.IOException;
-import java.net.URI;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
-import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest;
-import com.amazonaws.services.securitytoken.model.GetSessionTokenResult;
-import com.amazonaws.services.securitytoken.model.Credentials;
-
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.apache.hadoop.fs.s3native.S3xLoginHelper;
-import org.apache.hadoop.conf.Configuration;
-
-import org.junit.Test;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-
-/**
- * Tests use of temporary credentials (for example, AWS STS & S3).
- * This test extends a class that "does things to the root directory", and
- * should only be used against transient filesystems where you don't care about
- * the data.
- */
-public class TestS3ATemporaryCredentials extends AbstractFSContractTestBase {
- public static final String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled";
- public static final String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3ATemporaryCredentials.class);
-
- private static final String PROVIDER_CLASS
- = TemporaryAWSCredentialsProvider.NAME;
-
- private static final long TEST_FILE_SIZE = 1024;
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- /**
- * Test use of STS for requesting temporary credentials.
- *
- * The property test.sts.endpoint can be set to point this at different
- * STS endpoints. This test will use the AWS credentials (if provided) for
- * S3A tests to request temporary credentials, then attempt to use those
- * credentials instead.
- *
- * @throws IOException
- */
- @Test
- public void testSTS() throws IOException {
- Configuration conf = getContract().getConf();
- if (!conf.getBoolean(TEST_STS_ENABLED, true)) {
- skip("STS functional tests disabled");
- }
-
- S3xLoginHelper.Login login = S3AUtils.getAWSAccessKeys(
- URI.create("s3a://foobar"), conf);
- if (!login.hasLogin()) {
- skip("testSTS disabled because AWS credentials not configured");
- }
- AWSCredentialsProvider parentCredentials = new BasicAWSCredentialsProvider(
- login.getUser(), login.getPassword());
-
- String stsEndpoint = conf.getTrimmed(TEST_STS_ENDPOINT, "");
- AWSSecurityTokenServiceClient stsClient;
- stsClient = new AWSSecurityTokenServiceClient(parentCredentials);
- if (!stsEndpoint.isEmpty()) {
- LOG.debug("STS Endpoint ={}", stsEndpoint);
- stsClient.setEndpoint(stsEndpoint);
- }
- GetSessionTokenRequest sessionTokenRequest = new GetSessionTokenRequest();
- sessionTokenRequest.setDurationSeconds(900);
- GetSessionTokenResult sessionTokenResult;
- sessionTokenResult = stsClient.getSessionToken(sessionTokenRequest);
- Credentials sessionCreds = sessionTokenResult.getCredentials();
-
- String childAccessKey = sessionCreds.getAccessKeyId();
- conf.set(ACCESS_KEY, childAccessKey);
- String childSecretKey = sessionCreds.getSecretAccessKey();
- conf.set(SECRET_KEY, childSecretKey);
- String sessionToken = sessionCreds.getSessionToken();
- conf.set(SESSION_TOKEN, sessionToken);
-
- conf.set(AWS_CREDENTIALS_PROVIDER, PROVIDER_CLASS);
-
- try(S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
- createAndVerifyFile(fs, path("testSTS"), TEST_FILE_SIZE);
- }
-
- // now create an invalid set of credentials by changing the session
- // token
- conf.set(SESSION_TOKEN, "invalid-" + sessionToken);
- try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
- createAndVerifyFile(fs, path("testSTSInvalidToken"), TEST_FILE_SIZE);
- fail("Expected an access exception, but file access to "
- + fs.getUri() + " was allowed: " + fs);
- } catch (AWSS3IOException ex) {
- LOG.info("Expected Exception: {}", ex.toString());
- LOG.debug("Expected Exception: {}", ex, ex);
- }
- }
-
- @Test
- public void testTemporaryCredentialValidation() throws Throwable {
- Configuration conf = new Configuration();
- conf.set(ACCESS_KEY, "accesskey");
- conf.set(SECRET_KEY, "secretkey");
- conf.set(SESSION_TOKEN, "");
- TemporaryAWSCredentialsProvider provider
- = new TemporaryAWSCredentialsProvider(getFileSystem().getUri(), conf);
- try {
- AWSCredentials credentials = provider.getCredentials();
- fail("Expected a CredentialInitializationException,"
- + " got " + credentials);
- } catch (CredentialInitializationException expected) {
- // expected
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java
new file mode 100644
index 0000000..7e4273a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import org.apache.hadoop.fs.TestFileContext;
+
+/**
+ * Implementation of TestFileContext for S3a.
+ */
+public class ITestS3AFileContext extends TestFileContext{
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
new file mode 100644
index 0000000..4b8d4bb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+
+/**
+ * Extends FileContextCreateMkdirBaseTest for a S3a FileContext.
+ */
+public class ITestS3AFileContextCreateMkdir
+ extends FileContextCreateMkdirBaseTest {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
new file mode 100644
index 0000000..7247c16
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * S3A implementation of FileContextMainOperationsBaseTest.
+ */
+public class ITestS3AFileContextMainOperations
+ extends FileContextMainOperationsBaseTest {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+ @Override
+ protected boolean listCorruptedBlocksSupported() {
+ return false;
+ }
+
+ @Test
+ @Ignore
+ public void testCreateFlagAppendExistingFile() throws IOException {
+ //append not supported, so test removed
+ }
+
+ @Test
+ @Ignore
+ public void testCreateFlagCreateAppendExistingFile() throws IOException {
+ //append not supported, so test removed
+ }
+
+ @Test
+ @Ignore
+ public void testSetVerifyChecksum() throws IOException {
+ //checksums ignored, so test removed
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
new file mode 100644
index 0000000..5e88aba
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.net.URI;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FCStatisticsBaseTest;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+/**
+ * S3a implementation of FCStatisticsBaseTest.
+ */
+public class ITestS3AFileContextStatistics extends FCStatisticsBaseTest {
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"),
+ FileContext.DEFAULT_PERM, true);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true);
+ }
+
+ @Override
+ protected void verifyReadBytes(FileSystem.Statistics stats) {
+ // one blockSize for read, one for pread
+ Assert.assertEquals(2 * blockSize, stats.getBytesRead());
+ }
+
+ @Override
+ protected void verifyWrittenBytes(FileSystem.Statistics stats) {
+ //No extra bytes are written
+ Assert.assertEquals(blockSize, stats.getBytesWritten());
+ }
+
+ @Override
+ protected URI getFsUri() {
+ return fc.getHomeDirectory().toUri();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
new file mode 100644
index 0000000..fff1fcb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextURIBase;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * S3a implementation of FileContextURIBase.
+ */
+public class ITestS3AFileContextURI extends FileContextURIBase {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc1 = S3ATestUtils.createTestFileContext(conf);
+ fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
+ super.setUp();
+ }
+
+ @Test
+ @Ignore
+ public void testFileStatus() throws IOException {
+ // test disabled
+ // (the statistics tested with this method are not relevant for an S3FS)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java
new file mode 100644
index 0000000..d0312ba
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextUtilBase;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+
+/**
+ * S3A implementation of FileContextUtilBase.
+ */
+public class ITestS3AFileContextUtil extends FileContextUtilBase {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by cd...@apache.org.
HADOOP-13446. Support running isolated unit tests separate from AWS integration tests. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f9c346e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f9c346e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f9c346e
Branch: refs/heads/HDFS-9806
Commit: 6f9c346e577325ec2059d83d5636b5ff7fa6cdce
Parents: f0efea4
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Aug 23 07:18:49 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Aug 23 07:18:49 2016 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 5 +
hadoop-tools/hadoop-aws/pom.xml | 95 +++-
.../src/site/markdown/tools/hadoop-aws/index.md | 67 ++-
.../fs/contract/s3a/ITestS3AContractCreate.java | 35 ++
.../fs/contract/s3a/ITestS3AContractDelete.java | 34 ++
.../fs/contract/s3a/ITestS3AContractDistCp.java | 46 ++
.../s3a/ITestS3AContractGetFileStatus.java | 51 ++
.../fs/contract/s3a/ITestS3AContractMkdir.java | 34 ++
.../fs/contract/s3a/ITestS3AContractOpen.java | 34 ++
.../fs/contract/s3a/ITestS3AContractRename.java | 62 +++
.../contract/s3a/ITestS3AContractRootDir.java | 72 +++
.../fs/contract/s3a/ITestS3AContractSeek.java | 34 ++
.../fs/contract/s3a/TestS3AContractCreate.java | 33 --
.../fs/contract/s3a/TestS3AContractDelete.java | 31 --
.../fs/contract/s3a/TestS3AContractDistCp.java | 46 --
.../s3a/TestS3AContractGetFileStatus.java | 47 --
.../fs/contract/s3a/TestS3AContractMkdir.java | 34 --
.../fs/contract/s3a/TestS3AContractOpen.java | 31 --
.../fs/contract/s3a/TestS3AContractRename.java | 61 ---
.../fs/contract/s3a/TestS3AContractRootDir.java | 72 ---
.../fs/contract/s3a/TestS3AContractSeek.java | 31 --
.../fs/contract/s3n/ITestS3NContractCreate.java | 41 ++
.../fs/contract/s3n/ITestS3NContractDelete.java | 34 ++
.../fs/contract/s3n/ITestS3NContractMkdir.java | 34 ++
.../fs/contract/s3n/ITestS3NContractOpen.java | 34 ++
.../fs/contract/s3n/ITestS3NContractRename.java | 35 ++
.../contract/s3n/ITestS3NContractRootDir.java | 35 ++
.../fs/contract/s3n/ITestS3NContractSeek.java | 34 ++
.../fs/contract/s3n/TestS3NContractCreate.java | 38 --
.../fs/contract/s3n/TestS3NContractDelete.java | 31 --
.../fs/contract/s3n/TestS3NContractMkdir.java | 34 --
.../fs/contract/s3n/TestS3NContractOpen.java | 31 --
.../fs/contract/s3n/TestS3NContractRename.java | 32 --
.../fs/contract/s3n/TestS3NContractRootDir.java | 35 --
.../fs/contract/s3n/TestS3NContractSeek.java | 31 --
.../ITestBlockingThreadPoolExecutorService.java | 182 +++++++
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 250 +++++++++
.../fs/s3a/ITestS3ABlockingThreadPool.java | 80 +++
.../apache/hadoop/fs/s3a/ITestS3ABlocksize.java | 96 ++++
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 435 +++++++++++++++
.../hadoop/fs/s3a/ITestS3ACredentialsInURL.java | 155 ++++++
.../hadoop/fs/s3a/ITestS3AEncryption.java | 104 ++++
.../ITestS3AEncryptionAlgorithmPropagation.java | 83 +++
.../s3a/ITestS3AEncryptionFastOutputStream.java | 35 ++
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 192 +++++++
.../hadoop/fs/s3a/ITestS3AFastOutputStream.java | 74 +++
.../fs/s3a/ITestS3AFileOperationCost.java | 191 +++++++
.../fs/s3a/ITestS3AFileSystemContract.java | 106 ++++
.../fs/s3a/ITestS3ATemporaryCredentials.java | 148 +++++
.../TestBlockingThreadPoolExecutorService.java | 182 -------
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 250 ---------
.../fs/s3a/TestS3ABlockingThreadPool.java | 80 ---
.../apache/hadoop/fs/s3a/TestS3ABlocksize.java | 93 ----
.../hadoop/fs/s3a/TestS3AConfiguration.java | 429 ---------------
.../hadoop/fs/s3a/TestS3ACredentialsInURL.java | 155 ------
.../apache/hadoop/fs/s3a/TestS3AEncryption.java | 104 ----
.../TestS3AEncryptionAlgorithmPropagation.java | 82 ---
.../s3a/TestS3AEncryptionFastOutputStream.java | 35 --
.../hadoop/fs/s3a/TestS3AFailureHandling.java | 194 -------
.../hadoop/fs/s3a/TestS3AFastOutputStream.java | 74 ---
.../hadoop/fs/s3a/TestS3AFileOperationCost.java | 191 -------
.../fs/s3a/TestS3AFileSystemContract.java | 104 ----
.../fs/s3a/TestS3ATemporaryCredentials.java | 148 -----
.../fs/s3a/fileContext/ITestS3AFileContext.java | 23 +
.../ITestS3AFileContextCreateMkdir.java | 35 ++
.../ITestS3AFileContextMainOperations.java | 60 +++
.../ITestS3AFileContextStatistics.java | 61 +++
.../s3a/fileContext/ITestS3AFileContextURI.java | 44 ++
.../fileContext/ITestS3AFileContextUtil.java | 34 ++
.../fs/s3a/fileContext/TestS3AFileContext.java | 23 -
.../TestS3AFileContextCreateMkdir.java | 35 --
.../TestS3AFileContextMainOperations.java | 60 ---
.../TestS3AFileContextStatistics.java | 60 ---
.../s3a/fileContext/TestS3AFileContextURI.java | 43 --
.../s3a/fileContext/TestS3AFileContextUtil.java | 34 --
.../s3a/scale/ITestS3ADeleteFilesOneByOne.java | 44 ++
.../fs/s3a/scale/ITestS3ADeleteManyFiles.java | 135 +++++
.../s3a/scale/ITestS3ADirectoryPerformance.java | 187 +++++++
.../scale/ITestS3AInputStreamPerformance.java | 534 +++++++++++++++++++
.../fs/s3a/scale/TestS3ADeleteManyFiles.java | 135 -----
.../s3a/scale/TestS3ADirectoryPerformance.java | 187 -------
.../scale/TestS3AInputStreamPerformance.java | 534 -------------------
.../org/apache/hadoop/fs/s3a/yarn/ITestS3A.java | 88 +++
.../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 148 +++++
.../org/apache/hadoop/fs/s3a/yarn/TestS3A.java | 85 ---
.../fs/s3a/yarn/TestS3AMiniYarnCluster.java | 144 -----
...ITestInMemoryNativeS3FileSystemContract.java | 33 ++
.../ITestJets3tNativeFileSystemStore.java | 133 +++++
.../ITestJets3tNativeS3FileSystemContract.java | 33 ++
.../TestInMemoryNativeS3FileSystemContract.java | 30 --
.../TestJets3tNativeFileSystemStore.java | 129 -----
.../TestJets3tNativeS3FileSystemContract.java | 30 --
92 files changed, 4503 insertions(+), 4304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c3146ca..65e9672 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1193,6 +1193,11 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${maven-failsafe-plugin.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>${maven-install-plugin.version}</version>
</plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 8147d37..82a8581 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -46,7 +46,7 @@
</file>
</activation>
<properties>
- <maven.test.skip>true</maven.test.skip>
+ <skipITs>true</skipITs>
</properties>
</profile>
<profile>
@@ -57,11 +57,16 @@
</file>
</activation>
<properties>
- <maven.test.skip>false</maven.test.skip>
+ <skipITs>false</skipITs>
</properties>
</profile>
<profile>
<id>parallel-tests</id>
+ <activation>
+ <property>
+ <name>parallel-tests</name>
+ </property>
+ </activation>
<build>
<plugins>
<plugin>
@@ -96,12 +101,32 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkCount>${testsThreadCount}</forkCount>
+ <reuseForks>false</reuseForks>
+ <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+ <systemPropertyVariables>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+
+ <!-- Due to a Maven quirk, setting this to just -->
+ <!-- surefire.forkNumber won't do the parameter -->
+ <!-- substitution. Putting a prefix in front of it like -->
+ <!-- "fork-" makes it work. -->
+ <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
- <id>default-test</id>
- <phase>test</phase>
+ <id>default-integration-test</id>
<goals>
- <goal>test</goal>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
</goals>
<configuration>
<forkCount>${testsThreadCount}</forkCount>
@@ -129,32 +154,35 @@
<!-- Exclude all of these tests from parallel execution, -->
<!-- and instead run them sequentially in a separate -->
<!-- Surefire execution step later. -->
+ <includes>
+ <include>**/ITest*.java</include>
+ </includes>
<excludes>
- <exclude>**/TestJets3tNativeS3FileSystemContract.java</exclude>
- <exclude>**/TestS3ABlockingThreadPool.java</exclude>
- <exclude>**/TestS3AFastOutputStream.java</exclude>
- <exclude>**/TestS3AFileSystemContract.java</exclude>
- <exclude>**/TestS3AMiniYarnCluster.java</exclude>
- <exclude>**/Test*Root*.java</exclude>
+ <exclude>**/ITestJets3tNativeS3FileSystemContract.java</exclude>
+ <exclude>**/ITestS3ABlockingThreadPool.java</exclude>
+ <exclude>**/ITestS3AFastOutputStream.java</exclude>
+ <exclude>**/ITestS3AFileSystemContract.java</exclude>
+ <exclude>**/ITestS3AMiniYarnCluster.java</exclude>
+ <exclude>**/ITest*Root*.java</exclude>
</excludes>
</configuration>
</execution>
<execution>
- <id>sequential-tests</id>
- <phase>test</phase>
+ <id>sequential-integration-tests</id>
<goals>
- <goal>test</goal>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
</goals>
<configuration>
<!-- Do a sequential run for tests that cannot handle -->
<!-- parallel execution. -->
<includes>
- <include>**/TestJets3tNativeS3FileSystemContract.java</include>
- <include>**/TestS3ABlockingThreadPool.java</include>
- <include>**/TestS3AFastOutputStream.java</include>
- <include>**/TestS3AFileSystemContract.java</include>
- <include>**/TestS3AMiniYarnCluster.java</include>
- <include>**/Test*Root*.java</include>
+ <include>**/ITestJets3tNativeS3FileSystemContract.java</include>
+ <include>**/ITestS3ABlockingThreadPool.java</include>
+ <include>**/ITestS3AFastOutputStream.java</include>
+ <include>**/ITestS3AFileSystemContract.java</include>
+ <include>**/ITestS3AMiniYarnCluster.java</include>
+ <include>**/ITest*Root*.java</include>
</includes>
</configuration>
</execution>
@@ -163,6 +191,33 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>sequential-tests</id>
+ <activation>
+ <property>
+ <name>!parallel-tests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 78c9f12..5100030 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -1319,11 +1319,18 @@ works with S3 to something lower. See [AWS documentation](http://docs.aws.amazon
## Testing the S3 filesystem clients
-Due to eventual consistency, tests may fail without reason. Transient
-failures, which no longer occur upon rerunning the test, should thus be ignored.
+This module includes both unit tests, which can run in isolation without
+connecting to the S3 service, and integration tests, which require a working
+connection to S3 to interact with a bucket. Unit test suites follow the naming
+convention `Test*.java`. Integration tests follow the naming convention
+`ITest*.java`.
-To test the S3* filesystem clients, you need to provide two files
-which pass in authentication details to the test runner
+Due to eventual consistency, integration tests may fail without reason.
+Transient failures, which no longer occur upon rerunning the test, should thus
+be ignored.
+
+To integration test the S3* filesystem clients, you need to provide two files
+which pass in authentication details to the test runner.
1. `auth-keys.xml`
1. `core-site.xml`
@@ -1343,7 +1350,8 @@ need to apply a specific, non-default property change during the tests.
The presence of this file triggers the testing of the S3 classes.
-Without this file, *none of the tests in this module will be executed*
+Without this file, *none of the integration tests in this module will be
+executed*.
The XML file must contain all the ID/key information needed to connect
each of the filesystem clients to the object stores, and a URL for
@@ -1496,23 +1504,50 @@ source code tree, it is not going to get accidentally committed.
After completing the configuration, execute the test run through Maven.
- mvn clean test
+ mvn clean verify
-It's also possible to execute multiple test suites in parallel by enabling the
-`parallel-tests` Maven profile. The tests spend most of their time blocked on
-network I/O with the S3 service, so running in parallel tends to complete full
-test runs faster.
+It's also possible to execute multiple test suites in parallel by passing the
+`parallel-tests` property on the command line. The tests spend most of their
+time blocked on network I/O with the S3 service, so running in parallel tends to
+complete full test runs faster.
- mvn -Pparallel-tests clean test
+ mvn -Dparallel-tests clean verify
Some tests must run with exclusive access to the S3 bucket, so even with the
-`parallel-tests` profile enabled, several test suites will run in serial in a
-separate Maven execution step after the parallel tests.
+`parallel-tests` property, several test suites will run in serial in a separate
+Maven execution step after the parallel tests.
+
+By default, `parallel-tests` runs 4 test suites concurrently. This can be tuned
+by passing the `testsThreadCount` property.
+
+ mvn -Dparallel-tests -DtestsThreadCount=8 clean verify
+
+To run just unit tests, which do not require S3 connectivity or AWS credentials,
+use any of the above invocations, but switch the goal to `test` instead of
+`verify`.
+
+ mvn clean test
+
+ mvn -Dparallel-tests clean test
+
+ mvn -Dparallel-tests -DtestsThreadCount=8 clean test
+
+To run only a specific named subset of tests, pass the `test` property for unit
+tests or the `it.test` property for integration tests.
+
+ mvn clean test -Dtest=TestS3AInputPolicies
+
+ mvn clean verify -Dit.test=ITestS3AFileContextStatistics
-By default, the `parallel-tests` profile runs 4 test suites concurrently. This
-can be tuned by passing the `testsThreadCount` argument.
+ mvn clean verify -Dtest=TestS3A* -Dit.test=ITestS3A*
- mvn -Pparallel-tests -DtestsThreadCount=8 clean test
+Note that when running a specific subset of tests, the patterns passed in `test`
+and `it.test` override the configuration of which tests need to run in isolation
+in a separate serial phase (mentioned above). This can cause unpredictable
+results, so the recommendation is to avoid passing `parallel-tests` in
+combination with `test` or `it.test`. If you know that you are specifying only
+tests that can run safely in parallel, then it will work. For wide patterns,
+like `ITestS3A*` shown above, it may cause unpredictable test failures.
### Testing against different regions
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
new file mode 100644
index 0000000..d2a858f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests creating files.
+ */
+public class ITestS3AContractCreate extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java
new file mode 100644
index 0000000..a47dcae
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests covering deletes.
+ */
+public class ITestS3AContractDelete extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
new file mode 100644
index 0000000..28278fe
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import static org.apache.hadoop.fs.s3a.Constants.MIN_MULTIPART_THRESHOLD;
+import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_SIZE;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+
+/**
+ * Contract test suite covering S3A integration with DistCp.
+ */
+public class ITestS3AContractDistCp extends AbstractContractDistCpTest {
+
+ private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ newConf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_SETTING);
+ newConf.setLong(MULTIPART_SIZE, MULTIPART_SETTING);
+ return newConf;
+ }
+
+ @Override
+ protected S3AContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
new file mode 100644
index 0000000..c7ed5a3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+
+/**
+ * S3A contract tests covering getFileStatus.
+ */
+public class ITestS3AContractGetFileStatus
+ extends AbstractContractGetFileStatusTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ getLog().info("FS details {}", getFileSystem());
+ super.teardown();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ // aggressively low page size forces tests to go multipage
+ conf.setInt(Constants.MAX_PAGING_KEYS, 2);
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
new file mode 100644
index 0000000..d953e7e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test dir operations on S3A.
+ */
+public class ITestS3AContractMkdir extends AbstractContractMkdirTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java
new file mode 100644
index 0000000..a7bdc0d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests opening files.
+ */
+public class ITestS3AContractOpen extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java
new file mode 100644
index 0000000..5dba03d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
+
+/**
+ * S3A contract tests covering rename.
+ */
+public class ITestS3AContractRename extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Override
+ public void testRenameDirIntoExistingDir() throws Throwable {
+ describe("Verify renaming a dir into an existing dir puts the files"
+ +" from the source dir into the existing dir"
+ +" and leaves existing files alone");
+ FileSystem fs = getFileSystem();
+ String sourceSubdir = "source";
+ Path srcDir = path(sourceSubdir);
+ Path srcFilePath = new Path(srcDir, "source-256.txt");
+ byte[] srcDataset = dataset(256, 'a', 'z');
+ writeDataset(fs, srcFilePath, srcDataset, srcDataset.length, 1024, false);
+ Path destDir = path("dest");
+
+ Path destFilePath = new Path(destDir, "dest-512.txt");
+ byte[] destDateset = dataset(512, 'A', 'Z');
+ writeDataset(fs, destFilePath, destDateset, destDateset.length, 1024,
+ false);
+ assertIsFile(destFilePath);
+
+ boolean rename = fs.rename(srcDir, destDir);
+ assertFalse("s3a doesn't support rename to non-empty directory", rename);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java
new file mode 100644
index 0000000..8383a77
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * root dir operations against an S3 bucket.
+ */
+public class ITestS3AContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AContractRootDir.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Override
+ public void testListEmptyRootDirectory() throws IOException {
+ for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
+ try {
+ super.testListEmptyRootDirectory();
+ break;
+ } catch (AssertionError | FileNotFoundException e) {
+ if (attempt < maxAttempts) {
+ LOG.info("Attempt {} of {} for empty root directory test failed. "
+ + "This is likely caused by eventual consistency of S3 "
+ + "listings. Attempting retry.", attempt, maxAttempts);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e2) {
+ Thread.currentThread().interrupt();
+ fail("Test interrupted.");
+ break;
+ }
+ } else {
+ LOG.error(
+ "Empty root directory test failed {} attempts. Failing test.",
+ maxAttempts);
+ throw e;
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
new file mode 100644
index 0000000..1572fbc
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests covering file seek.
+ */
+public class ITestS3AContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
deleted file mode 100644
index 035252b..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-
-public class TestS3AContractCreate extends AbstractContractCreateTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java
deleted file mode 100644
index 733a517..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3AContractDelete extends AbstractContractDeleteTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java
deleted file mode 100644
index 7eb0afa..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import static org.apache.hadoop.fs.s3a.Constants.MIN_MULTIPART_THRESHOLD;
-import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_SIZE;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
-
-/**
- * Contract test suite covering S3A integration with DistCp.
- */
-public class TestS3AContractDistCp extends AbstractContractDistCpTest {
-
- private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB
-
- @Override
- protected Configuration createConfiguration() {
- Configuration newConf = super.createConfiguration();
- newConf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_SETTING);
- newConf.setLong(MULTIPART_SIZE, MULTIPART_SETTING);
- return newConf;
- }
-
- @Override
- protected S3AContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
deleted file mode 100644
index 5937d49..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-
-public class TestS3AContractGetFileStatus extends AbstractContractGetFileStatusTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Override
- public void teardown() throws Exception {
- getLog().info("FS details {}", getFileSystem());
- super.teardown();
- }
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- S3ATestUtils.disableFilesystemCaching(conf);
- // aggressively low page size forces tests to go multipage
- conf.setInt(Constants.MAX_PAGING_KEYS, 2);
- return conf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java
deleted file mode 100644
index a312782..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * Test dir operations on S3
- */
-public class TestS3AContractMkdir extends AbstractContractMkdirTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java
deleted file mode 100644
index f735deb..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3AContractOpen extends AbstractContractOpenTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java
deleted file mode 100644
index af1ed37..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.Test;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
-
-public class TestS3AContractRename extends AbstractContractRenameTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Override
- public void testRenameDirIntoExistingDir() throws Throwable {
- describe("Verify renaming a dir into an existing dir puts the files"
- +" from the source dir into the existing dir"
- +" and leaves existing files alone");
- FileSystem fs = getFileSystem();
- String sourceSubdir = "source";
- Path srcDir = path(sourceSubdir);
- Path srcFilePath = new Path(srcDir, "source-256.txt");
- byte[] srcDataset = dataset(256, 'a', 'z');
- writeDataset(fs, srcFilePath, srcDataset, srcDataset.length, 1024, false);
- Path destDir = path("dest");
-
- Path destFilePath = new Path(destDir, "dest-512.txt");
- byte[] destDateset = dataset(512, 'A', 'Z');
- writeDataset(fs, destFilePath, destDateset, destDateset.length, 1024,
- false);
- assertIsFile(destFilePath);
-
- boolean rename = fs.rename(srcDir, destDir);
- assertFalse("s3a doesn't support rename to non-empty directory", rename);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
deleted file mode 100644
index 1cf1eb8..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * root dir operations against an S3 bucket.
- */
-public class TestS3AContractRootDir extends
- AbstractContractRootDirectoryTest {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AContractRootDir.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Override
- public void testListEmptyRootDirectory() throws IOException {
- for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
- try {
- super.testListEmptyRootDirectory();
- break;
- } catch (AssertionError | FileNotFoundException e) {
- if (attempt < maxAttempts) {
- LOG.info("Attempt {} of {} for empty root directory test failed. "
- + "This is likely caused by eventual consistency of S3 "
- + "listings. Attempting retry.", attempt, maxAttempts);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e2) {
- Thread.currentThread().interrupt();
- fail("Test interrupted.");
- break;
- }
- } else {
- LOG.error(
- "Empty root directory test failed {} attempts. Failing test.",
- maxAttempts);
- throw e;
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
deleted file mode 100644
index d677ec4..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3AContractSeek extends AbstractContractSeekTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractCreate.java
new file mode 100644
index 0000000..502cf5a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractCreate.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+/**
+ * S3N contract tests creating files.
+ */
+public class ITestS3NContractCreate extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+
+ @Override
+ public void testOverwriteEmptyDirectory() throws Throwable {
+ ContractTestUtils.skip(
+ "blobstores can't distinguish empty directories from files");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractDelete.java
new file mode 100644
index 0000000..675f979
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractDelete.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests covering deletes.
+ */
+public class ITestS3NContractDelete extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractMkdir.java
new file mode 100644
index 0000000..3c566f3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractMkdir.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test dir operations on S3.
+ */
+public class ITestS3NContractMkdir extends AbstractContractMkdirTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractOpen.java
new file mode 100644
index 0000000..7ebfc4e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractOpen.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3N contract tests opening files.
+ */
+public class ITestS3NContractOpen extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRename.java
new file mode 100644
index 0000000..effe9eb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRename.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3N contract tests covering rename.
+ */
+public class ITestS3NContractRename extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRootDir.java
new file mode 100644
index 0000000..3fdf868
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRootDir.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Root dir operations against an S3 bucket.
+ */
+public class ITestS3NContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractSeek.java
new file mode 100644
index 0000000..9e1ce73
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractSeek.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3N contract tests covering file seek.
+ */
+public class ITestS3NContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
deleted file mode 100644
index e44e2b1..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-
-public class TestS3NContractCreate extends AbstractContractCreateTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-
- @Override
- public void testOverwriteEmptyDirectory() throws Throwable {
- ContractTestUtils.skip(
- "blobstores can't distinguish empty directories from files");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
deleted file mode 100644
index 1b79d27..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractDelete extends AbstractContractDeleteTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
deleted file mode 100644
index 527a31d..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * Test dir operations on S3
- */
-public class TestS3NContractMkdir extends AbstractContractMkdirTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
deleted file mode 100644
index 2186f28..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractOpen extends AbstractContractOpenTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
deleted file mode 100644
index d673416..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractRename extends AbstractContractRenameTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
deleted file mode 100644
index 94f8483..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * root dir operations against an S3 bucket
- */
-public class TestS3NContractRootDir extends
- AbstractContractRootDirectoryTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
deleted file mode 100644
index 6d04fff..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractSeek extends AbstractContractSeekTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-4685. Disable AM blacklisting
by default to mitigate situations that application get hanged. (Rohith Sharma
K S via wangda)
Posted by cd...@apache.org.
YARN-4685. Disable AM blacklisting by default to mitigate situations that application get hanged. (Rohith Sharma K S via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2da32a6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2da32a6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2da32a6e
Branch: refs/heads/HDFS-9806
Commit: 2da32a6ef9edebd86ca9672d10ce35b5a46818cc
Parents: 444b2ea
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Aug 19 16:31:16 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Aug 19 16:31:16 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2da32a6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 770d139..46e3323 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2699,7 +2699,7 @@ public class YarnConfiguration extends Configuration {
RM_PREFIX + "am-scheduling.node-blacklisting-enabled";
@Private
public static final boolean DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_ENABLED =
- true;
+ false;
@Private
/**
@@ -2709,7 +2709,7 @@ public class YarnConfiguration extends Configuration {
RM_PREFIX + "am-scheduling.node-blacklisting-disable-threshold";
@Private
public static final float
- DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD = 0.8f;
+ DEFAULT_AM_SCHEDULING_NODE_BLACKLISTING_DISABLE_THRESHOLD = 0.2f;
private static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX =
NM_NODE_LABELS_PROVIDER_PREFIX + "script.";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDFS-10711. Optimize
FSPermissionChecker group membership check. Contributed by Daryn Sharp.
Posted by cd...@apache.org.
HDFS-10711. Optimize FSPermissionChecker group membership check. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2550371f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2550371f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2550371f
Branch: refs/heads/HDFS-9806
Commit: 2550371f66c49fe0e40aadaa68744311270084ce
Parents: 091dd19
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Aug 19 09:12:17 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Aug 19 09:12:17 2016 -0500
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirAttrOp.java | 2 +-
.../server/namenode/FSPermissionChecker.java | 23 ++++++--------------
2 files changed, 8 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2550371f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index e6d36b8..e19341c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -88,7 +88,7 @@ public class FSDirAttrOp {
if (username != null && !pc.getUser().equals(username)) {
throw new AccessControlException("Non-super user cannot change owner");
}
- if (group != null && !pc.containsGroup(group)) {
+ if (group != null && !pc.isMemberOfGroup(group)) {
throw new AccessControlException("User does not belong to " + group);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2550371f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index 726319f..c9b1c76 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -17,10 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.Collection;
import java.util.Stack;
import org.apache.commons.logging.Log;
@@ -81,7 +78,7 @@ class FSPermissionChecker implements AccessControlEnforcer {
private final UserGroupInformation callerUgi;
private final String user;
- private final Set<String> groups;
+ private final Collection<String> groups;
private final boolean isSuper;
private final INodeAttributeProvider attributeProvider;
@@ -92,15 +89,13 @@ class FSPermissionChecker implements AccessControlEnforcer {
this.fsOwner = fsOwner;
this.supergroup = supergroup;
this.callerUgi = callerUgi;
- HashSet<String> s =
- new HashSet<String>(Arrays.asList(callerUgi.getGroupNames()));
- groups = Collections.unmodifiableSet(s);
+ this.groups = callerUgi.getGroups();
user = callerUgi.getShortUserName();
isSuper = user.equals(fsOwner) || groups.contains(supergroup);
this.attributeProvider = attributeProvider;
}
- public boolean containsGroup(String group) {
+ public boolean isMemberOfGroup(String group) {
return groups.contains(group);
}
@@ -108,10 +103,6 @@ class FSPermissionChecker implements AccessControlEnforcer {
return user;
}
- public Set<String> getGroups() {
- return groups;
- }
-
public boolean isSuperUser() {
return isSuper;
}
@@ -337,7 +328,7 @@ class FSPermissionChecker implements AccessControlEnforcer {
final FsAction checkAction;
if (getUser().equals(inode.getUserName())) { //user class
checkAction = mode.getUserAction();
- } else if (getGroups().contains(inode.getGroupName())) { //group class
+ } else if (isMemberOfGroup(inode.getGroupName())) { //group class
checkAction = mode.getGroupAction();
} else { //other class
checkAction = mode.getOtherAction();
@@ -407,7 +398,7 @@ class FSPermissionChecker implements AccessControlEnforcer {
// member of multiple groups that have entries that grant access, then
// it doesn't matter which is chosen, so exit early after first match.
String group = name == null ? inode.getGroupName() : name;
- if (getGroups().contains(group)) {
+ if (isMemberOfGroup(group)) {
FsAction masked = AclEntryStatusFormat.getPermission(entry).and(
mode.getGroupAction());
if (masked.implies(access)) {
@@ -470,7 +461,7 @@ class FSPermissionChecker implements AccessControlEnforcer {
&& mode.getUserAction().implies(access)) {
return;
}
- if (getGroups().contains(pool.getGroupName())
+ if (isMemberOfGroup(pool.getGroupName())
&& mode.getGroupAction().implies(access)) {
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HADOOP-12765. HttpServer2 should
switch to using the non-blocking SslSelectChannelConnector to prevent
performance degradation when handling SSL connections. Contributed by Min
Shen.
Posted by cd...@apache.org.
HADOOP-12765. HttpServer2 should switch to using the non-blocking SslSelectChannelConnector to prevent performance degradation when handling SSL connections. Contributed by Min Shen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03a9343d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03a9343d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03a9343d
Branch: refs/heads/HDFS-9806
Commit: 03a9343d5798384b66fbd21e1e028acaf55b00e9
Parents: 2550371
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Fri Aug 19 09:22:49 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Fri Aug 19 09:22:49 2016 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/pom.xml | 5 ++
.../org/apache/hadoop/http/HttpServer2.java | 76 +++++++++++---------
.../ssl/SslSelectChannelConnectorSecure.java | 58 +++++++++++++++
.../security/ssl/SslSocketConnectorSecure.java | 58 ---------------
.../hadoop/crypto/key/kms/server/MiniKMS.java | 9 +--
.../org/apache/hadoop/test/TestJettyHelper.java | 6 +-
hadoop-project/pom.xml | 5 ++
7 files changed, 118 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a9343d/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index ef2fdf5..c28a05c 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -106,6 +106,11 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-sslengine</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
<scope>runtime</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a9343d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 8199c9b..a2bb18f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -56,7 +56,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
-import org.apache.hadoop.security.ssl.SslSocketConnectorSecure;
+import org.apache.hadoop.security.ssl.SslSelectChannelConnectorSecure;
import org.apache.hadoop.jmx.JMXJsonServlet;
import org.apache.hadoop.log.LogLevel;
import org.apache.hadoop.security.SecurityUtil;
@@ -77,7 +77,7 @@ import org.mortbay.jetty.handler.ContextHandlerCollection;
import org.mortbay.jetty.handler.HandlerCollection;
import org.mortbay.jetty.handler.RequestLogHandler;
import org.mortbay.jetty.nio.SelectChannelConnector;
-import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.security.SslSelectChannelConnector;
import org.mortbay.jetty.servlet.AbstractSessionManager;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.DefaultServlet;
@@ -332,29 +332,7 @@ public final class HttpServer2 implements FilterContainer {
if ("http".equals(scheme)) {
listener = HttpServer2.createDefaultChannelConnector();
} else if ("https".equals(scheme)) {
- SslSocketConnector c = new SslSocketConnectorSecure();
- c.setHeaderBufferSize(1024*64);
- c.setNeedClientAuth(needsClientAuth);
- c.setKeyPassword(keyPassword);
-
- if (keyStore != null) {
- c.setKeystore(keyStore);
- c.setKeystoreType(keyStoreType);
- c.setPassword(keyStorePassword);
- }
-
- if (trustStore != null) {
- c.setTruststore(trustStore);
- c.setTruststoreType(trustStoreType);
- c.setTrustPassword(trustStorePassword);
- }
-
- if(null != excludeCiphers && !excludeCiphers.isEmpty()) {
- c.setExcludeCipherSuites(excludeCiphers.split(","));
- LOG.info("Excluded Cipher List:" + excludeCiphers);
- }
-
- listener = c;
+ listener = createHttpsChannelConnector();
} else {
throw new HadoopIllegalArgumentException(
@@ -367,6 +345,32 @@ public final class HttpServer2 implements FilterContainer {
server.loadListeners();
return server;
}
+
+ private Connector createHttpsChannelConnector() {
+ SslSelectChannelConnector c = new SslSelectChannelConnectorSecure();
+ configureChannelConnector(c);
+
+ c.setNeedClientAuth(needsClientAuth);
+ c.setKeyPassword(keyPassword);
+
+ if (keyStore != null) {
+ c.setKeystore(keyStore);
+ c.setKeystoreType(keyStoreType);
+ c.setPassword(keyStorePassword);
+ }
+
+ if (trustStore != null) {
+ c.setTruststore(trustStore);
+ c.setTruststoreType(trustStoreType);
+ c.setTrustPassword(trustStorePassword);
+ }
+
+ if(null != excludeCiphers && !excludeCiphers.isEmpty()) {
+ c.setExcludeCipherSuites(excludeCiphers.split(","));
+ LOG.info("Excluded Cipher List:" + excludeCiphers);
+ }
+ return c;
+ }
}
private HttpServer2(final Builder b) throws IOException {
@@ -508,21 +512,25 @@ public final class HttpServer2 implements FilterContainer {
Collections.<String, String> emptyMap(), new String[] { "/*" });
}
- @InterfaceAudience.Private
- public static Connector createDefaultChannelConnector() {
- SelectChannelConnector ret = new SelectChannelConnector();
- ret.setLowResourceMaxIdleTime(10000);
- ret.setAcceptQueueSize(128);
- ret.setResolveNames(false);
- ret.setUseDirectBuffers(false);
+ private static void configureChannelConnector(SelectChannelConnector c) {
+ c.setLowResourceMaxIdleTime(10000);
+ c.setAcceptQueueSize(128);
+ c.setResolveNames(false);
+ c.setUseDirectBuffers(false);
if(Shell.WINDOWS) {
// result of setting the SO_REUSEADDR flag is different on Windows
// http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
// without this 2 NN's can start on the same machine and listen on
// the same port with indeterminate routing of incoming requests to them
- ret.setReuseAddress(false);
+ c.setReuseAddress(false);
}
- ret.setHeaderBufferSize(1024*64);
+ c.setHeaderBufferSize(1024*64);
+ }
+
+ @InterfaceAudience.Private
+ public static Connector createDefaultChannelConnector() {
+ SelectChannelConnector ret = new SelectChannelConnector();
+ configureChannelConnector(ret);
return ret;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a9343d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SslSelectChannelConnectorSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SslSelectChannelConnectorSecure.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SslSelectChannelConnectorSecure.java
new file mode 100644
index 0000000..7de689b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SslSelectChannelConnectorSecure.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.ssl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import javax.net.ssl.SSLEngine;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.mortbay.jetty.security.SslSelectChannelConnector;
+
+/**
+ * This subclass of the Jetty SslSelectChannelConnector exists solely to
+ * control the TLS protocol versions allowed. This is fallout from the
+ * POODLE vulnerability (CVE-2014-3566), which requires that SSLv3 be disabled.
+ * Only TLS 1.0 and later protocols are allowed.
+ */
+@InterfaceAudience.Private
+public class SslSelectChannelConnectorSecure extends SslSelectChannelConnector {
+
+ public SslSelectChannelConnectorSecure() {
+ super();
+ }
+
+ /**
+ * Disable SSLv3 protocol.
+ */
+ @Override
+ protected SSLEngine createSSLEngine() throws IOException {
+ SSLEngine engine = super.createSSLEngine();
+ ArrayList<String> nonSSLProtocols = new ArrayList<String>();
+ for (String p : engine.getEnabledProtocols()) {
+ if (!p.contains("SSLv3")) {
+ nonSSLProtocols.add(p);
+ }
+ }
+ engine.setEnabledProtocols(nonSSLProtocols.toArray(
+ new String[nonSSLProtocols.size()]));
+ return engine;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a9343d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SslSocketConnectorSecure.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SslSocketConnectorSecure.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SslSocketConnectorSecure.java
deleted file mode 100644
index 52ab7ad..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SslSocketConnectorSecure.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.security.ssl;
-
-import org.mortbay.jetty.security.SslSocketConnector;
-
-import javax.net.ssl.SSLServerSocket;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.util.ArrayList;
-
-/**
- * This subclass of the Jetty SslSocketConnector exists solely to control
- * the TLS protocol versions allowed. This is fallout from the POODLE
- * vulnerability (CVE-2014-3566), which requires that SSLv3 be disabled.
- * Only TLS 1.0 and later protocols are allowed.
- */
-public class SslSocketConnectorSecure extends SslSocketConnector {
-
- public SslSocketConnectorSecure() {
- super();
- }
-
- /**
- * Create a new ServerSocket that will not accept SSLv3 connections,
- * but will accept TLSv1.x connections.
- */
- protected ServerSocket newServerSocket(String host, int port,int backlog)
- throws IOException {
- SSLServerSocket socket = (SSLServerSocket)
- super.newServerSocket(host, port, backlog);
- ArrayList<String> nonSSLProtocols = new ArrayList<String>();
- for (String p : socket.getEnabledProtocols()) {
- if (!p.contains("SSLv3")) {
- nonSSLProtocols.add(p);
- }
- }
- socket.setEnabledProtocols(nonSSLProtocols.toArray(
- new String[nonSSLProtocols.size()]));
- return socket;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a9343d/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index f520edf..292d731 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -18,15 +18,16 @@
package org.apache.hadoop.crypto.key.kms.server;
import com.google.common.base.Preconditions;
+
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.ssl.SslSocketConnectorSecure;
import org.apache.hadoop.util.ThreadUtil;
+import org.apache.hadoop.security.ssl.SslSelectChannelConnectorSecure;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
-import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.security.SslSelectChannelConnector;
import org.mortbay.jetty.webapp.WebAppContext;
import java.io.File;
@@ -54,7 +55,7 @@ public class MiniKMS {
if (!ssl) {
server.getConnectors()[0].setHost(host);
} else {
- SslSocketConnector c = new SslSocketConnectorSecure();
+ SslSelectChannelConnector c = new SslSelectChannelConnectorSecure();
c.setHost(host);
c.setNeedClientAuth(false);
c.setKeystore(keyStore);
@@ -71,7 +72,7 @@ public class MiniKMS {
private static URL getJettyURL(Server server) {
boolean ssl = server.getConnectors()[0].getClass()
- == SslSocketConnectorSecure.class;
+ == SslSelectChannelConnectorSecure.class;
try {
String scheme = (ssl) ? "https" : "http";
return new URL(scheme + "://" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a9343d/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java
index 4c17aa7..a9b661e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestJettyHelper.java
@@ -24,14 +24,14 @@ import java.net.ServerSocket;
import java.net.URL;
import java.net.UnknownHostException;
-import org.apache.hadoop.security.ssl.SslSocketConnectorSecure;
+import org.apache.hadoop.security.ssl.SslSelectChannelConnectorSecure;
import org.junit.Test;
import org.junit.rules.MethodRule;
import org.junit.runners.model.FrameworkMethod;
import org.junit.runners.model.Statement;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
-import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.security.SslSelectChannelConnector;
public class TestJettyHelper implements MethodRule {
private boolean ssl;
@@ -93,7 +93,7 @@ public class TestJettyHelper implements MethodRule {
server.getConnectors()[0].setHost(host);
server.getConnectors()[0].setPort(port);
} else {
- SslSocketConnector c = new SslSocketConnectorSecure();
+ SslSelectChannelConnector c = new SslSelectChannelConnectorSecure();
c.setHost(host);
c.setPort(port);
c.setNeedClientAuth(false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a9343d/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index dee79f7..c3146ca 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -527,6 +527,11 @@
<version>${jetty.version}</version>
</dependency>
<dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-sslengine</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>7.0.55</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-5533. JMX AM Used metrics for
queue wrong when app submited to nodelabel partition (Bibin A Chundatt via
Varun Saxena)
Posted by cd...@apache.org.
YARN-5533. JMX AM Used metrics for queue wrong when app submited to nodelabel partition (Bibin A Chundatt via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/091dd19e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/091dd19e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/091dd19e
Branch: refs/heads/HDFS-9806
Commit: 091dd19e8636b8bd5668dbb3bcd8bbce7c952274
Parents: 8aed3741
Author: Varun Saxena <va...@apache.org>
Authored: Fri Aug 19 17:30:17 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Aug 19 17:30:17 2016 +0530
----------------------------------------------------------------------
.../resourcemanager/scheduler/capacity/LeafQueue.java | 3 ++-
.../capacity/TestNodeLabelContainerAllocation.java | 9 +++++++++
2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/091dd19e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 6bbe85e..636762f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -774,7 +774,8 @@ public class LeafQueue extends AbstractCSQueue {
application.getAMResource(partitionName));
user.getResourceUsage().decAMUsed(partitionName,
application.getAMResource(partitionName));
- metrics.decAMUsed(application.getUser(), application.getAMResource());
+ metrics.decAMUsed(application.getUser(),
+ application.getAMResource(partitionName));
}
applicationAttemptMap.remove(application.getApplicationAttemptId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/091dd19e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 9070577..5ccb6f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
@@ -1959,6 +1960,14 @@ public class TestNodeLabelContainerAllocation {
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
assertEquals(0 * GB, leafQueue.getMetrics().getAvailableMB());
assertEquals(5 * GB, leafQueue.getMetrics().getAllocatedMB());
+
+ // Kill all apps in queue a
+ cs.killAllAppsInQueue("a");
+ rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
+ rm1.waitForAppRemovedFromScheduler(app1.getApplicationId());
+
+ assertEquals(0 * GB, leafQueue.getMetrics().getUsedAMResourceMB());
+ assertEquals(0, leafQueue.getMetrics().getUsedAMResourceVCores());
rm1.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: Revert "HDFS-10762. Pass IIP for
file status related methods. Contributed by Daryn Sharp."
Posted by cd...@apache.org.
Revert "HDFS-10762. Pass IIP for file status related methods. Contributed by Daryn Sharp."
This reverts commit 22fc46d7659972ff016ccf1c6f781f0c160be26f.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ca4d6dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ca4d6dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ca4d6dd
Branch: refs/heads/HDFS-9806
Commit: 3ca4d6ddfd199c95677721ff3bcb95d1da45bd88
Parents: f4d4d34
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Aug 22 16:57:45 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Aug 22 16:57:45 2016 -0500
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirAppendOp.java | 6 +-
.../server/namenode/FSDirStatAndListingOp.java | 80 +++++++++++---------
.../hdfs/server/namenode/FSDirWriteFileOp.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 14 ++--
.../hdfs/server/namenode/INodesInPath.java | 42 ++--------
.../hadoop/hdfs/TestReservedRawPaths.java | 21 -----
6 files changed, 64 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 5192352..3a5d7dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -85,10 +85,9 @@ final class FSDirAppendOp {
final LocatedBlock lb;
final FSDirectory fsd = fsn.getFSDirectory();
final String src;
- final INodesInPath iip;
fsd.writeLock();
try {
- iip = fsd.resolvePathForWrite(pc, srcArg);
+ final INodesInPath iip = fsd.resolvePathForWrite(pc, srcArg);
src = iip.getPath();
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
@@ -149,7 +148,8 @@ final class FSDirAppendOp {
fsd.writeUnlock();
}
- HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, iip);
+ HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, src, false,
+ FSDirectory.isReservedRawName(srcArg));
if (lb != null) {
NameNode.stateChangeLog.debug(
"DIR* NameSystem.appendFile: file {} for {} at {} block {} block"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 88be510..c9eedf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -108,16 +108,16 @@ class FSDirStatAndListingOp {
if (!DFSUtil.isValidName(src)) {
throw new InvalidPathException("Invalid file name: " + src);
}
- final INodesInPath iip;
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- iip = fsd.resolvePath(pc, srcArg, resolveLink);
+ final INodesInPath iip = fsd.resolvePath(pc, srcArg, resolveLink);
+ src = iip.getPath();
fsd.checkPermission(pc, iip, false, null, null, null, null, false);
} else {
src = FSDirectory.resolvePath(srcArg, fsd);
- iip = fsd.getINodesInPath(src, resolveLink);
}
- return getFileInfo(fsd, iip);
+ return getFileInfo(fsd, src, FSDirectory.isReservedRawName(srcArg),
+ resolveLink);
}
/**
@@ -230,6 +230,7 @@ class FSDirStatAndListingOp {
String src, byte[] startAfter, boolean needLocation, boolean isSuperUser)
throws IOException {
String srcs = FSDirectory.normalizePath(src);
+ final boolean isRawPath = FSDirectory.isReservedRawName(src);
if (FSDirectory.isExactReservedName(srcs)) {
return getReservedListing(fsd);
}
@@ -256,7 +257,7 @@ class FSDirStatAndListingOp {
return new DirectoryListing(
new HdfsFileStatus[]{ createFileStatus(
fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- needLocation, parentStoragePolicy, iip)
+ needLocation, parentStoragePolicy, snapshot, isRawPath, iip)
}, 0);
}
@@ -281,7 +282,7 @@ class FSDirStatAndListingOp {
cur.getLocalNameBytes());
listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), nodeAttrs,
needLocation, getStoragePolicyID(curPolicy, parentStoragePolicy),
- iipWithChild);
+ snapshot, isRawPath, iipWithChild);
listingCnt++;
if (needLocation) {
// Once we hit lsLimit locations, stop.
@@ -338,6 +339,7 @@ class FSDirStatAndListingOp {
listing[i] = createFileStatus(
fsd, sRoot.getLocalNameBytes(), nodeAttrs,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
+ Snapshot.CURRENT_STATE_ID, false,
INodesInPath.fromINode(sRoot));
}
return new DirectoryListing(
@@ -361,8 +363,10 @@ class FSDirStatAndListingOp {
* @return object containing information regarding the file
* or null if file not found
*/
- static HdfsFileStatus getFileInfo(FSDirectory fsd,
- INodesInPath iip, boolean includeStoragePolicy) throws IOException {
+ static HdfsFileStatus getFileInfo(
+ FSDirectory fsd, String path, INodesInPath iip, boolean isRawPath,
+ boolean includeStoragePolicy)
+ throws IOException {
fsd.readLock();
try {
final INode node = iip.getLastINode();
@@ -373,32 +377,36 @@ class FSDirStatAndListingOp {
byte policyId = includeStoragePolicy && !node.isSymlink() ?
node.getStoragePolicyID() :
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
- INodeAttributes nodeAttrs = getINodeAttributes(fsd, iip.getPath(),
+ INodeAttributes nodeAttrs = getINodeAttributes(fsd, path,
HdfsFileStatus.EMPTY_NAME,
node, iip.getPathSnapshotId());
return createFileStatus(fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- policyId, iip);
+ policyId, iip.getPathSnapshotId(), isRawPath, iip);
} finally {
fsd.readUnlock();
}
}
- static HdfsFileStatus getFileInfo(FSDirectory fsd, INodesInPath iip)
+ static HdfsFileStatus getFileInfo(
+ FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath)
throws IOException {
- if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
- return FSDirectory.DOT_RESERVED_STATUS;
- }
-
- if (iip.isDotSnapshotDir()) {
- if (fsd.getINode4DotSnapshot(iip) != null) {
- return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
- HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
- HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
+ fsd.readLock();
+ try {
+ HdfsFileStatus status = null;
+ final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
+ if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
+ status = FSDirectory.DOT_RESERVED_STATUS;
+ } else if (iip.isDotSnapshotDir()) {
+ if (fsd.getINode4DotSnapshot(iip) != null) {
+ status = FSDirectory.DOT_SNAPSHOT_DIR_STATUS;
+ }
+ } else {
+ status = getFileInfo(fsd, src, iip, isRawPath, true);
}
- return null;
+ return status;
+ } finally {
+ fsd.readUnlock();
}
-
- return getFileInfo(fsd, iip, true);
}
/**
@@ -415,12 +423,15 @@ class FSDirStatAndListingOp {
*/
private static HdfsFileStatus createFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- boolean needLocation, byte storagePolicy, INodesInPath iip)
+ boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath,
+ INodesInPath iip)
throws IOException {
if (needLocation) {
- return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
+ return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy,
+ snapshot, isRawPath, iip);
} else {
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
+ snapshot, isRawPath, iip);
}
}
@@ -434,7 +445,8 @@ class FSDirStatAndListingOp {
INodesInPath iip) throws IOException {
INodeAttributes nodeAttrs = getINodeAttributes(
fsd, fullPath, path, iip.getLastINode(), snapshot);
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
+ snapshot, isRawPath, iip);
}
/**
@@ -442,15 +454,14 @@ class FSDirStatAndListingOp {
* @param iip the INodesInPath containing the target INode and its ancestors
*/
static HdfsFileStatus createFileStatus(
- FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- byte storagePolicy, INodesInPath iip) throws IOException {
+ FSDirectory fsd, byte[] path,
+ INodeAttributes nodeAttrs, byte storagePolicy, int snapshot,
+ boolean isRawPath, INodesInPath iip) throws IOException {
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
final boolean isEncrypted;
final INode node = iip.getLastINode();
- final int snapshot = iip.getPathSnapshotId();
- final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
@@ -500,9 +511,10 @@ class FSDirStatAndListingOp {
* Create FileStatus with location info by file INode
* @param iip the INodesInPath containing the target INode and its ancestors
*/
- private static HdfsFileStatus createLocatedFileStatus(
+ private static HdfsLocatedFileStatus createLocatedFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- byte storagePolicy, INodesInPath iip) throws IOException {
+ byte storagePolicy, int snapshot,
+ boolean isRawPath, INodesInPath iip) throws IOException {
assert fsd.hasReadLock();
long size = 0; // length is zero for directories
short replication = 0;
@@ -510,8 +522,6 @@ class FSDirStatAndListingOp {
LocatedBlocks loc = null;
final boolean isEncrypted;
final INode node = iip.getLastINode();
- final int snapshot = iip.getPathSnapshotId();
- final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 030d8cc..8429024 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -340,6 +340,7 @@ class FSDirWriteFileOp {
version = ezInfo.protocolVersion;
}
+ boolean isRawPath = FSDirectory.isReservedRawName(src);
FSDirectory fsd = fsn.getFSDirectory();
INodesInPath iip = fsd.resolvePathForWrite(pc, src);
src = iip.getPath();
@@ -443,7 +444,7 @@ class FSDirWriteFileOp {
NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
src + " inode " + newNode.getId() + " " + holder);
}
- return FSDirStatAndListingOp.getFileInfo(fsd, iip);
+ return FSDirStatAndListingOp.getFileInfo(fsd, src, false, isRawPath);
}
static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 5dc5a03..b6652e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -531,24 +531,21 @@ public class FSDirectory implements Closeable {
* @throws FileNotFoundException
* @throws AccessControlException
*/
- @VisibleForTesting
- public INodesInPath resolvePath(FSPermissionChecker pc, String src)
+ INodesInPath resolvePath(FSPermissionChecker pc, String src)
throws UnresolvedLinkException, FileNotFoundException,
AccessControlException {
return resolvePath(pc, src, true);
}
- @VisibleForTesting
- public INodesInPath resolvePath(FSPermissionChecker pc, String src,
+ INodesInPath resolvePath(FSPermissionChecker pc, String src,
boolean resolveLink) throws UnresolvedLinkException,
FileNotFoundException, AccessControlException {
byte[][] components = INode.getPathComponents(src);
- boolean isRaw = isReservedRawName(components);
- if (isPermissionEnabled && pc != null && isRaw) {
+ if (isPermissionEnabled && pc != null && isReservedRawName(components)) {
pc.checkSuperuserPrivilege();
}
components = resolveComponents(components, this);
- return INodesInPath.resolve(rootDir, components, isRaw, resolveLink);
+ return INodesInPath.resolve(rootDir, components, resolveLink);
}
INodesInPath resolvePathForWrite(FSPermissionChecker pc, String src)
@@ -1665,7 +1662,8 @@ public class FSDirectory implements Closeable {
HdfsFileStatus getAuditFileInfo(INodesInPath iip)
throws IOException {
return (namesystem.isAuditEnabled() && namesystem.isExternalInvocation())
- ? FSDirStatAndListingOp.getFileInfo(this, iip, false) : null;
+ ? FSDirStatAndListingOp.getFileInfo(this, iip.getPath(), iip, false,
+ false) : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index af8998f..86cab28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -126,12 +126,6 @@ public class INodesInPath {
static INodesInPath resolve(final INodeDirectory startingDir,
final byte[][] components, final boolean resolveLink)
throws UnresolvedLinkException {
- return resolve(startingDir, components, false, resolveLink);
- }
-
- static INodesInPath resolve(final INodeDirectory startingDir,
- final byte[][] components, final boolean isRaw,
- final boolean resolveLink) throws UnresolvedLinkException {
Preconditions.checkArgument(startingDir.compareTo(components[0]) == 0);
INode curNode = startingDir;
@@ -231,7 +225,7 @@ public class INodesInPath {
System.arraycopy(inodes, 0, newNodes, 0, newNodes.length);
inodes = newNodes;
}
- return new INodesInPath(inodes, components, isRaw, isSnapshot, snapshotId);
+ return new INodesInPath(inodes, components, isSnapshot, snapshotId);
}
private static boolean shouldUpdateLatestId(int sid, int snapshotId) {
@@ -255,8 +249,7 @@ public class INodesInPath {
INode[] inodes = new INode[iip.inodes.length];
System.arraycopy(iip.inodes, 0, inodes, 0, inodes.length);
inodes[pos] = inode;
- return new INodesInPath(inodes, iip.path, iip.isRaw,
- iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, iip.path, iip.isSnapshot, iip.snapshotId);
}
/**
@@ -274,8 +267,7 @@ public class INodesInPath {
byte[][] path = new byte[iip.path.length + 1][];
System.arraycopy(iip.path, 0, path, 0, path.length - 1);
path[path.length - 1] = childName;
- return new INodesInPath(inodes, path, iip.isRaw,
- iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, path, iip.isSnapshot, iip.snapshotId);
}
private final byte[][] path;
@@ -287,13 +279,6 @@ public class INodesInPath {
* true if this path corresponds to a snapshot
*/
private final boolean isSnapshot;
-
- /**
- * true if this is a /.reserved/raw path. path component resolution strips
- * it from the path so need to track it separately.
- */
- private final boolean isRaw;
-
/**
* For snapshot paths, it is the id of the snapshot; or
* {@link Snapshot#CURRENT_STATE_ID} if the snapshot does not exist. For
@@ -302,18 +287,17 @@ public class INodesInPath {
*/
private final int snapshotId;
- private INodesInPath(INode[] inodes, byte[][] path, boolean isRaw,
- boolean isSnapshot,int snapshotId) {
+ private INodesInPath(INode[] inodes, byte[][] path, boolean isSnapshot,
+ int snapshotId) {
Preconditions.checkArgument(inodes != null && path != null);
this.inodes = inodes;
this.path = path;
- this.isRaw = isRaw;
this.isSnapshot = isSnapshot;
this.snapshotId = snapshotId;
}
private INodesInPath(INode[] inodes, byte[][] path) {
- this(inodes, path, false, false, CURRENT_STATE_ID);
+ this(inodes, path, false, CURRENT_STATE_ID);
}
/**
@@ -416,7 +400,7 @@ public class INodesInPath {
final byte[][] apath = new byte[length][];
System.arraycopy(this.inodes, 0, anodes, 0, length);
System.arraycopy(this.path, 0, apath, 0, length);
- return new INodesInPath(anodes, apath, isRaw, false, snapshotId);
+ return new INodesInPath(anodes, apath, false, snapshotId);
}
/**
@@ -444,7 +428,7 @@ public class INodesInPath {
byte[][] existingPath = new byte[i][];
System.arraycopy(inodes, 0, existing, 0, i);
System.arraycopy(path, 0, existingPath, 0, i);
- return new INodesInPath(existing, existingPath, isRaw, false, snapshotId);
+ return new INodesInPath(existing, existingPath, false, snapshotId);
}
/**
@@ -454,20 +438,10 @@ public class INodesInPath {
return this.isSnapshot;
}
- /**
- * @return if .snapshot is the last path component.
- */
boolean isDotSnapshotDir() {
return isDotSnapshotDir(getLastLocalName());
}
- /**
- * @return if this is a /.reserved/raw path.
- */
- public boolean isRaw() {
- return isRaw;
- }
-
private static String toString(INode inode) {
return inode == null? null: inode.getLocalName();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
index c09d346..449f715 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
-import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
@@ -50,8 +49,6 @@ import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesNotEqual;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.apache.hadoop.test.GenericTestUtils.assertMatches;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestReservedRawPaths {
@@ -103,24 +100,6 @@ public class TestReservedRawPaths {
}
/**
- * Verify resolving path will return an iip that tracks if the original
- * path was a raw path.
- */
- @Test(timeout = 120000)
- public void testINodesInPath() throws IOException {
- FSDirectory fsd = cluster.getNamesystem().getFSDirectory();
- final String path = "/path";
-
- INodesInPath iip = fsd.resolvePath(null, path);
- assertFalse(iip.isRaw());
- assertEquals(path, iip.getPath());
-
- iip = fsd.resolvePath(null, "/.reserved/raw" + path);
- assertTrue(iip.isRaw());
- assertEquals(path, iip.getPath());
- }
-
- /**
* Basic read/write tests of raw files.
* Create a non-encrypted file
* Create an encryption zone
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDFS-8312. Added permission check
for moving file to Trash. (Weiwei Yang via Eric Yang)
Posted by cd...@apache.org.
HDFS-8312. Added permission check for moving file to Trash. (Weiwei Yang via Eric Yang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c49333be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c49333be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c49333be
Branch: refs/heads/HDFS-9806
Commit: c49333becfa7652460976a61eb86522010bcfeed
Parents: 4070caa
Author: Eric Yang <ey...@apache.org>
Authored: Mon Aug 22 18:29:56 2016 -0700
Committer: Eric Yang <ey...@apache.org>
Committed: Mon Aug 22 18:29:56 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/Options.java | 3 +-
.../apache/hadoop/fs/TrashPolicyDefault.java | 10 ++-
.../ClientNamenodeProtocolTranslatorPB.java | 7 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 1 +
...tNamenodeProtocolServerSideTranslatorPB.java | 14 +++-
.../hdfs/server/namenode/FSDirRenameOp.java | 28 +++++--
.../apache/hadoop/hdfs/TestDFSPermission.java | 81 ++++++++++++++++++++
7 files changed, 132 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index da75d1c..dc50286 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -213,7 +213,8 @@ public final class Options {
*/
public static enum Rename {
NONE((byte) 0), // No options
- OVERWRITE((byte) 1); // Overwrite the rename destination
+ OVERWRITE((byte) 1), // Overwrite the rename destination
+ TO_TRASH ((byte) 2); // Rename to trash
private final byte code;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index 14f4c0c..66ef890 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -106,6 +106,7 @@ public class TrashPolicyDefault extends TrashPolicy {
return deletionInterval != 0;
}
+ @SuppressWarnings("deprecation")
@Override
public boolean moveToTrash(Path path) throws IOException {
if (!isEnabled())
@@ -156,10 +157,11 @@ public class TrashPolicyDefault extends TrashPolicy {
trashPath = new Path(orig + Time.now());
}
- if (fs.rename(path, trashPath)) { // move to current trash
- LOG.info("Moved: '" + path + "' to trash at: " + trashPath);
- return true;
- }
+ // move to current trash
+ fs.rename(path, trashPath,
+ Rename.TO_TRASH);
+ LOG.info("Moved: '" + path + "' to trash at: " + trashPath);
+ return true;
} catch (IOException e) {
cause = e;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index bcf5269..57f8fd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -523,16 +523,21 @@ public class ClientNamenodeProtocolTranslatorPB implements
public void rename2(String src, String dst, Rename... options)
throws IOException {
boolean overwrite = false;
+ boolean toTrash = false;
if (options != null) {
for (Rename option : options) {
if (option == Rename.OVERWRITE) {
overwrite = true;
+ } else if (option == Rename.TO_TRASH) {
+ toTrash = true;
}
}
}
Rename2RequestProto req = Rename2RequestProto.newBuilder().
setSrc(src).
- setDst(dst).setOverwriteDest(overwrite).
+ setDst(dst).
+ setOverwriteDest(overwrite).
+ setMoveToTrash(toTrash).
build();
try {
if (Client.isAsynchronousMode()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index aac48a4..83f5828 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -245,6 +245,7 @@ message Rename2RequestProto {
required string src = 1;
required string dst = 2;
required bool overwriteDest = 3;
+ optional bool moveToTrash = 4;
}
message Rename2ResponseProto { // void response
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 7a4ff82..641cd20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.protocolPB;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
@@ -602,10 +603,21 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
@Override
public Rename2ResponseProto rename2(RpcController controller,
Rename2RequestProto req) throws ServiceException {
+ // resolve rename options
+ ArrayList<Rename> optionList = new ArrayList<Rename>();
+ if(req.getOverwriteDest()) {
+ optionList.add(Rename.OVERWRITE);
+ } else if(req.hasMoveToTrash()) {
+ optionList.add(Rename.TO_TRASH);
+ }
+
+ if(optionList.isEmpty()) {
+ optionList.add(Rename.NONE);
+ }
try {
server.rename2(req.getSrc(), req.getDst(),
- req.getOverwriteDest() ? Rename.OVERWRITE : Rename.NONE);
+ optionList.toArray(new Rename[optionList.size()]));
} catch (IOException e) {
throw new ServiceException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index d90139b..f98f8b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -261,11 +261,29 @@ class FSDirRenameOp {
src = srcIIP.getPath();
dst = dstIIP.getPath();
if (fsd.isPermissionEnabled()) {
- // Rename does not operate on link targets
- // Do not resolveLink when checking permissions of src and dst
- // Check write access to parent of src
- fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
- false);
+ boolean renameToTrash = false;
+ if (null != options &&
+ Arrays.asList(options).
+ contains(Options.Rename.TO_TRASH)) {
+ renameToTrash = true;
+ }
+
+ if(renameToTrash) {
+ // if destination is the trash directory,
+ // besides the permission check on "rename"
+ // we need to enforce the check for "delete"
+ // otherwise, it would expose a
+ // security hole that stuff moved to trash
+ // will be deleted by superuser
+ fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null,
+ FsAction.ALL, true);
+ } else {
+ // Rename does not operate on link targets
+ // Do not resolveLink when checking permissions of src and dst
+ // Check write access to parent of src
+ fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null,
+ null, false);
+ }
// Check write access to ancestor of dst
fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null, null,
false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
index e6524f3..d0d00e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.security.AccessControlException;
@@ -288,6 +289,86 @@ public class TestDFSPermission {
FsPermission.createImmutable((short)0777));
}
+ @Test(timeout=30000)
+ public void testTrashPermission() throws Exception {
+ // /BSS user1:group2 777
+ // /BSS/user1 user1:group2 755
+ // /BSS/user1/test user1:group1 600
+ Path rootDir = new Path("/BSS");
+ Path user1Dir = new Path("/BSS/user1");
+ Path user1File = new Path("/BSS/user1/test");
+
+ try {
+ conf.set(CommonConfigurationKeys.FS_TRASH_INTERVAL_KEY, "10");
+ fs = FileSystem.get(conf);
+
+ fs.mkdirs(rootDir);
+ fs.setPermission(rootDir, new FsPermission((short) 0777));
+
+ login(USER1);
+ fs.mkdirs(user1Dir);
+ fs.setPermission(user1Dir, new FsPermission((short) 0755));
+ fs.setOwner(user1Dir, USER1.getShortUserName(), GROUP2_NAME);
+
+ create(OpType.CREATE, user1File);
+ fs.setOwner(user1File, USER1.getShortUserName(), GROUP1_NAME);
+ fs.setPermission(user1File, new FsPermission((short) 0600));
+
+ try {
+ // login as user2, attempt to delete /BSS/user1
+ // this should fail because user2 has no permission to
+ // its sub directory.
+ login(USER2);
+ fs.delete(user1Dir, true);
+ fail("User2 should not be allowed to delete user1's dir.");
+ } catch (AccessControlException e) {
+ e.printStackTrace();
+ assertTrue("Permission denied messages must carry the username",
+ e.getMessage().contains(USER2_NAME));
+ }
+
+ // ensure the /BSS/user1 still exists
+ assertTrue(fs.exists(user1Dir));
+
+ try {
+ login(SUPERUSER);
+ Trash trash = new Trash(fs, conf);
+ Path trashRoot = trash.getCurrentTrashDir(user1Dir);
+ while(true) {
+ trashRoot = trashRoot.getParent();
+ if(trashRoot.getParent().isRoot()) {
+ break;
+ }
+ }
+ fs.mkdirs(trashRoot);
+ fs.setPermission(trashRoot, new FsPermission((short) 0777));
+
+ // login as user2, attempt to move /BSS/user1 to trash
+ // this should also fail otherwise the directory will be
+ // removed by trash emptier (emptier is running by superuser)
+ login(USER2);
+ Trash userTrash = new Trash(fs, conf);
+ assertTrue(userTrash.isEnabled());
+ userTrash.moveToTrash(user1Dir);
+ fail("User2 should not be allowed to move"
+ + "user1's dir to trash");
+ } catch (IOException e) {
+ // expect the exception is caused by permission denied
+ assertTrue(e.getCause() instanceof AccessControlException);
+ e.printStackTrace();
+ assertTrue("Permission denied messages must carry the username",
+ e.getCause().getMessage().contains(USER2_NAME));
+ }
+
+ // ensure /BSS/user1 still exists
+ assertEquals(fs.exists(user1Dir), true);
+ } finally {
+ login(SUPERUSER);
+ fs.delete(rootDir, true);
+ conf.set(CommonConfigurationKeys.FS_TRASH_INTERVAL_KEY, "0");
+ }
+ }
+
/* check if the ownership of a file/directory is set correctly */
@Test
public void testOwnership() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HADOOP-13504. Refactor jni_common
to conform to C89 restrictions imposed by Visual Studio 2010. Contributed by
Sammi Chen
Posted by cd...@apache.org.
HADOOP-13504. Refactor jni_common to conform to C89 restrictions imposed by Visual Studio 2010. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dbcaf999
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dbcaf999
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dbcaf999
Branch: refs/heads/HDFS-9806
Commit: dbcaf999d9ea7a7c6c090903d1982e5b61200c8b
Parents: c5c3e81
Author: Kai Zheng <ka...@intel.com>
Authored: Fri Aug 19 11:18:15 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Fri Aug 19 11:18:15 2016 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/io/erasurecode/jni_common.c | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dbcaf999/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
index 17e05db..2b1d9ea 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
@@ -43,21 +43,26 @@ void setCoder(JNIEnv* env, jobject thiz, IsalCoder* pCoder) {
}
IsalCoder* getCoder(JNIEnv* env, jobject thiz) {
- jclass clazz = (*env)->GetObjectClass(env, thiz);
+ jclass clazz;
+ jmethodID mid;
+ jboolean verbose;
+ jfieldID fid;
+ IsalCoder* pCoder;
- jmethodID mid = (*env)->GetMethodID(env, clazz, "allowVerboseDump", "()Z");
+ clazz = (*env)->GetObjectClass(env, thiz);
+ mid = (*env)->GetMethodID(env, clazz, "allowVerboseDump", "()Z");
if (mid == NULL) {
THROW(env, "java/lang/UnsatisfiedLinkError",
"Method allowVerboseDump not found");
}
- jboolean verbose = (*env)->CallBooleanMethod(env, thiz, mid);
+ verbose = (*env)->CallBooleanMethod(env, thiz, mid);
- jfieldID fid = (*env)->GetFieldID(env, clazz, "nativeCoder", "J");
+ fid = (*env)->GetFieldID(env, clazz, "nativeCoder", "J");
if (fid == NULL) {
THROW(env, "java/lang/UnsatisfiedLinkError",
"Field nativeCoder not found");
}
- IsalCoder* pCoder = (IsalCoder*)(*env)->GetLongField(env, thiz, fid);
+ pCoder = (IsalCoder*)(*env)->GetLongField(env, thiz, fid);
pCoder->verbose = (verbose == JNI_TRUE) ? 1 : 0;
return pCoder;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: MAPREDUCE-6587. Remove unused
params in connection-related methods of Fetcher. Contributed by Yiqun Lin.
Posted by cd...@apache.org.
MAPREDUCE-6587. Remove unused params in connection-related methods of Fetcher. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cc4a670
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cc4a670
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cc4a670
Branch: refs/heads/HDFS-9806
Commit: 8cc4a67059e37b2083cd5468b35a64a403a3e3ae
Parents: c49333b
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Aug 23 17:04:55 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Aug 23 17:04:55 2016 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/mapreduce/task/reduce/Fetcher.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc4a670/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index d8dd7b5..be2f84f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -263,7 +263,7 @@ class Fetcher<K,V> extends Thread {
DataInputStream input = null;
try {
- setupConnectionsWithRetry(host, remaining, url);
+ setupConnectionsWithRetry(url);
if (stopped) {
abortConnect(host, remaining);
} else {
@@ -374,9 +374,8 @@ class Fetcher<K,V> extends Thread {
}
}
- private void setupConnectionsWithRetry(MapHost host,
- Set<TaskAttemptID> remaining, URL url) throws IOException {
- openConnectionWithRetry(host, remaining, url);
+ private void setupConnectionsWithRetry(URL url) throws IOException {
+ openConnectionWithRetry(url);
if (stopped) {
return;
}
@@ -396,8 +395,7 @@ class Fetcher<K,V> extends Thread {
verifyConnection(url, msgToEncode, encHash);
}
- private void openConnectionWithRetry(MapHost host,
- Set<TaskAttemptID> remaining, URL url) throws IOException {
+ private void openConnectionWithRetry(URL url) throws IOException {
long startTime = Time.monotonicNow();
boolean shouldWait = true;
while (shouldWait) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HADOOP-13527. Add Spark to
CallerContext LimitedPrivate scope. (Contributed by Weiqing Yang)
Posted by cd...@apache.org.
HADOOP-13527. Add Spark to CallerContext LimitedPrivate scope. (Contributed by Weiqing Yang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/115ecb52
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/115ecb52
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/115ecb52
Branch: refs/heads/HDFS-9806
Commit: 115ecb52a86b49aad3d058a6b4c1c7926b8b0a40
Parents: d37b45d
Author: Mingliang Liu <li...@apache.org>
Authored: Sun Aug 21 09:40:29 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Sun Aug 21 09:40:29 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/ipc/CallerContext.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/115ecb52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index 3d21bfe..bdfa471 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -33,7 +33,7 @@ import java.util.Arrays;
* This class is immutable.
*/
@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "Hive", "MapReduce",
- "Pig", "YARN"})
+ "Pig", "Spark", "YARN"})
@InterfaceStability.Evolving
public final class CallerContext {
public static final Charset SIGNATURE_ENCODING = StandardCharsets.UTF_8;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-10692. Update JDiff report's
base version for HDFS from 2.6.0 to 2.7.2. Contributed by Wangda Tan.
Posted by cd...@apache.org.
HDFS-10692. Update JDiff report's base version for HDFS from 2.6.0 to 2.7.2. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc7a1c54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc7a1c54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc7a1c54
Branch: refs/heads/HDFS-9806
Commit: dc7a1c54f4447fb01980ae61d5d67e90c4f52f00
Parents: 115ecb5
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Mon Aug 22 11:35:18 2016 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Mon Aug 22 11:35:18 2016 -0700
----------------------------------------------------------------------
.../jdiff/Apache_Hadoop_HDFS_2.7.2.xml | 21505 +++++++++++++++++
1 file changed, 21505 insertions(+)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-5533. JMX AM Used metrics for
queue wrong when app submited to nodelabel partition (Bibin A Chundatt via
Varun Saxena)
Posted by cd...@apache.org.
YARN-5533. JMX AM Used metrics for queue wrong when app submited to nodelabel partition (Bibin A Chundatt via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59557e85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59557e85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59557e85
Branch: refs/heads/HDFS-9806
Commit: 59557e85a40fb91c7106e8ff3bfe958ffa244b29
Parents: 8179f9a
Author: Varun Saxena <va...@apache.org>
Authored: Fri Aug 19 15:01:48 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Aug 19 15:01:48 2016 +0530
----------------------------------------------------------------------
.../resourcemanager/scheduler/capacity/LeafQueue.java | 3 ++-
.../capacity/TestNodeLabelContainerAllocation.java | 9 +++++++++
2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59557e85/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 6bbe85e..636762f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -774,7 +774,8 @@ public class LeafQueue extends AbstractCSQueue {
application.getAMResource(partitionName));
user.getResourceUsage().decAMUsed(partitionName,
application.getAMResource(partitionName));
- metrics.decAMUsed(application.getUser(), application.getAMResource());
+ metrics.decAMUsed(application.getUser(),
+ application.getAMResource(partitionName));
}
applicationAttemptMap.remove(application.getApplicationAttemptId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59557e85/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 9070577..251e4dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
@@ -1959,6 +1960,14 @@ public class TestNodeLabelContainerAllocation {
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
assertEquals(0 * GB, leafQueue.getMetrics().getAvailableMB());
assertEquals(5 * GB, leafQueue.getMetrics().getAllocatedMB());
+
+ // Kill all apps in queue a
+ cs.killAllAppsInQueue("a");
+ rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
+ rm1.waitForAppRemovedFromScheduler(app1.getApplicationId());
+
+ assertEquals(0 * GB, leafQueue.getMetrics().getUsedAMResourceMB());
+ assertEquals(0 * GB, leafQueue.getMetrics().getUsedAMResourceVCores());
rm1.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
deleted file mode 100644
index a22dd28..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.yarn;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FsStatus;
-import org.apache.hadoop.fs.Path;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import java.util.EnumSet;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-public class TestS3A {
- private FileContext fc;
-
- @Rule
- public final Timeout testTimeout = new Timeout(90000);
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fc != null) {
- fc.delete(getTestPath(), true);
- }
- }
-
- protected Path getTestPath() {
- return new Path("/tests3afc");
- }
-
- @Test
- public void testS3AStatus() throws Exception {
- FsStatus fsStatus = fc.getFsStatus(null);
- assertNotNull(fsStatus);
- assertTrue("Used capacity should be positive: " + fsStatus.getUsed(),
- fsStatus.getUsed() >= 0);
- assertTrue("Remaining capacity should be positive: " + fsStatus
- .getRemaining(),
- fsStatus.getRemaining() >= 0);
- assertTrue("Capacity should be positive: " + fsStatus.getCapacity(),
- fsStatus.getCapacity() >= 0);
- }
-
- @Test
- public void testS3ACreateFileInSubDir() throws Exception {
- Path dirPath = getTestPath();
- fc.mkdir(dirPath,FileContext.DIR_DEFAULT_PERM,true);
- Path filePath = new Path(dirPath, "file");
- try (FSDataOutputStream file = fc.create(filePath, EnumSet.of(CreateFlag
- .CREATE))) {
- file.write(666);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java
deleted file mode 100644
index 990d79f..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.fs.s3a.yarn;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.examples.WordCount;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-
-import org.junit.After;
-import static org.junit.Assert.assertTrue;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
-public class TestS3AMiniYarnCluster {
-
- private final Configuration conf = new YarnConfiguration();
- private S3AFileSystem fs;
- private MiniYARNCluster yarnCluster;
- private final String rootPath = "/tests/MiniClusterWordCount/";
-
- @Before
- public void beforeTest() throws IOException {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fs.mkdirs(new Path(rootPath + "input/"));
-
- yarnCluster = new MiniYARNCluster("MiniClusterWordCount", // testName
- 1, // number of node managers
- 1, // number of local log dirs per node manager
- 1); // number of hdfs dirs per node manager
- yarnCluster.init(conf);
- yarnCluster.start();
- }
-
- @After
- public void afterTest() throws IOException {
- fs.delete(new Path(rootPath), true);
- yarnCluster.stop();
- }
-
- @Test
- public void testWithMiniCluster() throws Exception {
- Path input = new Path(rootPath + "input/in.txt");
- input = input.makeQualified(fs.getUri(), fs.getWorkingDirectory());
- Path output = new Path(rootPath + "output/");
- output = output.makeQualified(fs.getUri(), fs.getWorkingDirectory());
-
- writeStringToFile(input, "first line\nsecond line\nthird line");
-
- Job job = Job.getInstance(conf, "word count");
- job.setJarByClass(WordCount.class);
- job.setMapperClass(WordCount.TokenizerMapper.class);
- job.setCombinerClass(WordCount.IntSumReducer.class);
- job.setReducerClass(WordCount.IntSumReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- FileInputFormat.addInputPath(job, input);
- FileOutputFormat.setOutputPath(job, output);
-
- int exitCode = (job.waitForCompletion(true) ? 0 : 1);
- assertEquals("Returned error code.", 0, exitCode);
-
- assertTrue(fs.exists(new Path(output, "_SUCCESS")));
- String outputAsStr = readStringFromFile(new Path(output, "part-r-00000"));
- Map<String, Integer> resAsMap = getResultAsMap(outputAsStr);
-
- assertEquals(4, resAsMap.size());
- assertEquals(1, (int) resAsMap.get("first"));
- assertEquals(1, (int) resAsMap.get("second"));
- assertEquals(1, (int) resAsMap.get("third"));
- assertEquals(3, (int) resAsMap.get("line"));
- }
-
- /**
- * helper method
- */
- private Map<String, Integer> getResultAsMap(String outputAsStr) throws IOException {
- Map<String, Integer> result = new HashMap<>();
- for (String line : outputAsStr.split("\n")) {
- String[] tokens = line.split("\t");
- result.put(tokens[0], Integer.parseInt(tokens[1]));
- }
- return result;
- }
-
- /**
- * helper method
- */
- private void writeStringToFile(Path path, String string) throws IOException {
- FileContext fc = S3ATestUtils.createTestFileContext(conf);
- try (FSDataOutputStream file = fc.create(path,
- EnumSet.of(CreateFlag.CREATE))) {
- file.write(string.getBytes());
- }
- }
-
- /**
- * helper method
- */
- private String readStringFromFile(Path path) {
- try (FSDataInputStream in = fs.open(path)) {
- long bytesLen = fs.getFileStatus(path).getLen();
- byte[] buffer = new byte[(int) bytesLen];
- IOUtils.readFully(in, buffer, 0, buffer.length);
- return new String(buffer);
- } catch (IOException e) {
- throw new RuntimeException("Failed to read from [" + path + "]", e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestInMemoryNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestInMemoryNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestInMemoryNativeS3FileSystemContract.java
new file mode 100644
index 0000000..adbf950
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestInMemoryNativeS3FileSystemContract.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.IOException;
+
+/**
+ * S3N basic contract tests through mock in-memory S3 implementation.
+ */
+public class ITestInMemoryNativeS3FileSystemContract
+ extends NativeS3FileSystemContractBaseTest {
+
+ @Override
+ NativeFileSystemStore getNativeFileSystemStore() throws IOException {
+ return new InMemoryNativeFileSystemStore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeFileSystemStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeFileSystemStore.java
new file mode 100644
index 0000000..cfe622c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeFileSystemStore.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import static org.junit.Assert.*;
+import static org.junit.Assume.*;
+
+import org.junit.Before;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * S3N tests through live S3 service.
+ */
+public class ITestJets3tNativeFileSystemStore {
+ private Configuration conf;
+ private Jets3tNativeFileSystemStore store;
+ private NativeS3FileSystem fs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ store = new Jets3tNativeFileSystemStore();
+ fs = new NativeS3FileSystem(store);
+ conf.setBoolean("fs.s3n.multipart.uploads.enabled", true);
+ conf.setLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024);
+ fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ store.purge("test");
+ } catch (Exception e) {}
+ }
+
+ @BeforeClass
+ public static void checkSettings() throws Exception {
+ Configuration conf = new Configuration();
+ assumeNotNull(conf.get("fs.s3n.awsAccessKeyId"));
+ assumeNotNull(conf.get("fs.s3n.awsSecretAccessKey"));
+ assumeNotNull(conf.get("test.fs.s3n.name"));
+ }
+
+ protected void writeRenameReadCompare(Path path, long len)
+ throws IOException, NoSuchAlgorithmException {
+ // If len > fs.s3n.multipart.uploads.block.size,
+ // we'll use a multipart upload copy
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ OutputStream out = new BufferedOutputStream(
+ new DigestOutputStream(fs.create(path, false), digest));
+ for (long i = 0; i < len; i++) {
+ out.write('Q');
+ }
+ out.flush();
+ out.close();
+
+ assertTrue("Exists", fs.exists(path));
+
+ // Depending on if this file is over 5 GB or not,
+ // rename will cause a multipart upload copy
+ Path copyPath = path.suffix(".copy");
+ fs.rename(path, copyPath);
+
+ assertTrue("Copy exists", fs.exists(copyPath));
+
+ // Download file from S3 and compare the digest against the original
+ MessageDigest digest2 = MessageDigest.getInstance("MD5");
+ InputStream in = new BufferedInputStream(
+ new DigestInputStream(fs.open(copyPath), digest2));
+ long copyLen = 0;
+ while (in.read() != -1) {
+ copyLen++;
+ }
+ in.close();
+
+ assertEquals("Copy length matches original", len, copyLen);
+ assertArrayEquals("Digests match", digest.digest(), digest2.digest());
+ }
+
+ @Test
+ public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
+ // Regular upload, regular copy
+ writeRenameReadCompare(new Path("/test/small"), 16384);
+ }
+
+ @Test
+ public void testMediumUpload() throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, regular copy
+ writeRenameReadCompare(new Path("/test/medium"), 33554432); // 100 MB
+ }
+
+ /*
+ Enable Multipart upload to run this test
+ @Test
+ public void testExtraLargeUpload()
+ throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, multipart copy
+ writeRenameReadCompare(new Path("/test/xlarge"), 5368709121L); // 5GB+1byte
+ }
+ */
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeS3FileSystemContract.java
new file mode 100644
index 0000000..e51eaf6
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeS3FileSystemContract.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.IOException;
+
+/**
+ * S3N basic contract tests through live S3 service.
+ */
+public class ITestJets3tNativeS3FileSystemContract
+ extends NativeS3FileSystemContractBaseTest {
+
+ @Override
+ NativeFileSystemStore getNativeFileSystemStore() throws IOException {
+ return new Jets3tNativeFileSystemStore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
deleted file mode 100644
index 664d39e..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.IOException;
-
-public class TestInMemoryNativeS3FileSystemContract
- extends NativeS3FileSystemContractBaseTest {
-
- @Override
- NativeFileSystemStore getNativeFileSystemStore() throws IOException {
- return new InMemoryNativeFileSystemStore();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
deleted file mode 100644
index dbd476e..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import static org.junit.Assert.*;
-import static org.junit.Assume.*;
-
-import org.junit.Before;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.security.DigestInputStream;
-import java.security.DigestOutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-
-public class TestJets3tNativeFileSystemStore {
- private Configuration conf;
- private Jets3tNativeFileSystemStore store;
- private NativeS3FileSystem fs;
-
- @Before
- public void setUp() throws Exception {
- conf = new Configuration();
- store = new Jets3tNativeFileSystemStore();
- fs = new NativeS3FileSystem(store);
- conf.setBoolean("fs.s3n.multipart.uploads.enabled", true);
- conf.setLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024);
- fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
- }
-
- @After
- public void tearDown() throws Exception {
- try {
- store.purge("test");
- } catch (Exception e) {}
- }
-
- @BeforeClass
- public static void checkSettings() throws Exception {
- Configuration conf = new Configuration();
- assumeNotNull(conf.get("fs.s3n.awsAccessKeyId"));
- assumeNotNull(conf.get("fs.s3n.awsSecretAccessKey"));
- assumeNotNull(conf.get("test.fs.s3n.name"));
- }
-
- protected void writeRenameReadCompare(Path path, long len)
- throws IOException, NoSuchAlgorithmException {
- // If len > fs.s3n.multipart.uploads.block.size,
- // we'll use a multipart upload copy
- MessageDigest digest = MessageDigest.getInstance("MD5");
- OutputStream out = new BufferedOutputStream(
- new DigestOutputStream(fs.create(path, false), digest));
- for (long i = 0; i < len; i++) {
- out.write('Q');
- }
- out.flush();
- out.close();
-
- assertTrue("Exists", fs.exists(path));
-
- // Depending on if this file is over 5 GB or not,
- // rename will cause a multipart upload copy
- Path copyPath = path.suffix(".copy");
- fs.rename(path, copyPath);
-
- assertTrue("Copy exists", fs.exists(copyPath));
-
- // Download file from S3 and compare the digest against the original
- MessageDigest digest2 = MessageDigest.getInstance("MD5");
- InputStream in = new BufferedInputStream(
- new DigestInputStream(fs.open(copyPath), digest2));
- long copyLen = 0;
- while (in.read() != -1) {copyLen++;}
- in.close();
-
- assertEquals("Copy length matches original", len, copyLen);
- assertArrayEquals("Digests match", digest.digest(), digest2.digest());
- }
-
- @Test
- public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
- // Regular upload, regular copy
- writeRenameReadCompare(new Path("/test/small"), 16384);
- }
-
- @Test
- public void testMediumUpload() throws IOException, NoSuchAlgorithmException {
- // Multipart upload, regular copy
- writeRenameReadCompare(new Path("/test/medium"), 33554432); // 100 MB
- }
-
- /*
- Enable Multipart upload to run this test
- @Test
- public void testExtraLargeUpload()
- throws IOException, NoSuchAlgorithmException {
- // Multipart upload, multipart copy
- writeRenameReadCompare(new Path("/test/xlarge"), 5368709121L); // 5GB+1byte
- }
- */
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeS3FileSystemContract.java
deleted file mode 100644
index 42d6f06..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeS3FileSystemContract.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.IOException;
-
-public class TestJets3tNativeS3FileSystemContract
- extends NativeS3FileSystemContractBaseTest {
-
- @Override
- NativeFileSystemStore getNativeFileSystemStore() throws IOException {
- return new Jets3tNativeFileSystemStore();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HADOOP-13405 doc for
fs.s3a.acl.default indicates incorrect values. Contributed by Shen Yinjie
Posted by cd...@apache.org.
HADOOP-13405 doc for fs.s3a.acl.default indicates incorrect values. Contributed by Shen Yinjie
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/040c185d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/040c185d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/040c185d
Branch: refs/heads/HDFS-9806
Commit: 040c185d624a18627d23cedb12bf91a950ada2fc
Parents: 913a895
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 18 14:35:26 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 18 14:36:55 2016 +0100
----------------------------------------------------------------------
.../hadoop-common/src/main/resources/core-default.xml | 6 +++---
.../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 4 ++--
.../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 6 +++---
3 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/040c185d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index e78795c..dd8f9af 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1039,9 +1039,9 @@
<property>
<name>fs.s3a.acl.default</name>
- <description>Set a canned ACL for newly created and copied objects. Value may be private,
- public-read, public-read-write, authenticated-read, log-delivery-write,
- bucket-owner-read, or bucket-owner-full-control.</description>
+ <description>Set a canned ACL for newly created and copied objects. Value may be Private,
+ PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead,
+ or BucketOwnerFullControl.</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/040c185d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 612b648..6b8f9f5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -128,8 +128,8 @@ public final class Constants {
public static final String FAST_BUFFER_SIZE = "fs.s3a.fast.buffer.size";
public static final int DEFAULT_FAST_BUFFER_SIZE = 1048576; //1MB
- // private | public-read | public-read-write | authenticated-read |
- // log-delivery-write | bucket-owner-read | bucket-owner-full-control
+ // Private | PublicRead | PublicReadWrite | AuthenticatedRead |
+ // LogDeliveryWrite | BucketOwnerRead | BucketOwnerFullControl
public static final String CANNED_ACL = "fs.s3a.acl.default";
public static final String DEFAULT_CANNED_ACL = "";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/040c185d/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index cb1df83..d459fb5 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -554,9 +554,9 @@ this capability.
<property>
<name>fs.s3a.acl.default</name>
- <description>Set a canned ACL for newly created and copied objects. Value may be private,
- public-read, public-read-write, authenticated-read, log-delivery-write,
- bucket-owner-read, or bucket-owner-full-control.</description>
+ <description>Set a canned ACL for newly created and copied objects. Value may be Private,
+ PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead,
+ or BucketOwnerFullControl.</description>
</property>
<property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-4676. Automatic and
Asynchronous Decommissioning Nodes Status Tracking. Contributed by Diniel
Zhi. (cherry picked from commit d464483bf7f0b3e3be3ba32cd6c3eee546747ab5)
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 22aa0ee..5a89e54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -272,6 +272,11 @@ public class MockNodes {
@Override
public void setUntrackedTimeStamp(long timeStamp) {
}
+
+ @Override
+ public Integer getDecommissioningTimeout() {
+ return null;
+ }
};
private static RMNode buildRMNode(int rack, final Resource perNode,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 5856e59..f843261 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -709,6 +709,9 @@ public class MockRM extends ResourceManager {
public void waitForState(NodeId nodeId, NodeState finalState)
throws InterruptedException {
RMNode node = getRMContext().getRMNodes().get(nodeId);
+ if (node == null) {
+ node = getRMContext().getInactiveRMNodes().get(nodeId);
+ }
Assert.assertNotNull("node shouldn't be null", node);
int timeWaiting = 0;
while (!finalState.equals(node.getState())) {
@@ -722,11 +725,17 @@ public class MockRM extends ResourceManager {
timeWaiting += WAIT_MS_PER_LOOP;
}
- System.out.println("Node State is : " + node.getState());
+ System.out.println("Node " + nodeId + " State is : " + node.getState());
Assert.assertEquals("Node state is not correct (timedout)", finalState,
node.getState());
}
+ public void sendNodeEvent(MockNM nm, RMNodeEventType event) throws Exception {
+ RMNodeImpl node = (RMNodeImpl)
+ getRMContext().getRMNodes().get(nm.getNodeId());
+ node.handle(new RMNodeEvent(nm.getNodeId(), event));
+ }
+
public KillApplicationResponse killApp(ApplicationId appId) throws Exception {
ApplicationClientProtocol client = getClientRMService();
KillApplicationRequest req = KillApplicationRequest.newInstance(appId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
new file mode 100644
index 0000000..690de30
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.DecommissioningNodesWatcher.DecommissioningNodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests DecommissioningNodesWatcher.
+ */
+public class TestDecommissioningNodesWatcher {
+ private MockRM rm;
+
+ @Test
+ public void testDecommissioningNodesWatcher() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT, "40");
+
+ rm = new MockRM(conf);
+ rm.start();
+
+ DecommissioningNodesWatcher watcher =
+ new DecommissioningNodesWatcher(rm.getRMContext());
+
+ MockNM nm1 = rm.registerNode("host1:1234", 10240);
+ RMNode node1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+ NodeId id1 = nm1.getNodeId();
+
+ rm.waitForState(id1, NodeState.RUNNING);
+ Assert.assertFalse(watcher.checkReadyToBeDecommissioned(id1));
+
+ RMApp app = rm.submitApp(2000);
+ MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
+
+ // Setup nm1 as DECOMMISSIONING for DecommissioningNodesWatcher.
+ rm.sendNodeEvent(nm1, RMNodeEventType.GRACEFUL_DECOMMISSION);
+ rm.waitForState(id1, NodeState.DECOMMISSIONING);
+
+ // Update status with decreasing number of running containers until 0.
+ watcher.update(node1, createNodeStatus(id1, app, 12));
+ watcher.update(node1, createNodeStatus(id1, app, 11));
+ Assert.assertFalse(watcher.checkReadyToBeDecommissioned(id1));
+
+ watcher.update(node1, createNodeStatus(id1, app, 1));
+ Assert.assertEquals(DecommissioningNodeStatus.WAIT_CONTAINER,
+ watcher.checkDecommissioningStatus(id1));
+
+ watcher.update(node1, createNodeStatus(id1, app, 0));
+ Assert.assertEquals(DecommissioningNodeStatus.WAIT_APP,
+ watcher.checkDecommissioningStatus(id1));
+
+ // Set app to be FINISHED and verified DecommissioningNodeStatus is READY.
+ MockRM.finishAMAndVerifyAppState(app, rm, nm1, am);
+ rm.waitForState(app.getApplicationId(), RMAppState.FINISHED);
+ Assert.assertEquals(DecommissioningNodeStatus.READY,
+ watcher.checkDecommissioningStatus(id1));
+ }
+
+ @After
+ public void tearDown() {
+ if (rm != null) {
+ rm.stop();
+ }
+ }
+
+ private NodeStatus createNodeStatus(
+ NodeId nodeId, RMApp app, int numRunningContainers) {
+ return NodeStatus.newInstance(
+ nodeId, 0, getContainerStatuses(app, numRunningContainers),
+ new ArrayList<ApplicationId>(),
+ NodeHealthStatus.newInstance(
+ true, "", System.currentTimeMillis() - 1000),
+ null, null, null);
+ }
+
+ // Get mocked ContainerStatus for bunch of containers,
+ // where numRunningContainers are RUNNING.
+ private List<ContainerStatus> getContainerStatuses(
+ RMApp app, int numRunningContainers) {
+ // Total 12 containers
+ final int total = 12;
+ numRunningContainers = Math.min(total, numRunningContainers);
+ List<ContainerStatus> output = new ArrayList<ContainerStatus>();
+ for (int i = 0; i < total; i++) {
+ ContainerState cstate = (i >= numRunningContainers)?
+ ContainerState.COMPLETE : ContainerState.RUNNING;
+ output.add(ContainerStatus.newInstance(
+ ContainerId.newContainerId(
+ ApplicationAttemptId.newInstance(app.getApplicationId(), i), 1),
+ cstate, "Dummy", 0));
+ }
+ return output;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index 83a7c73..e82b93c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -254,17 +254,6 @@ public class TestRMNodeTransitions {
cm.getNumDecommissioningNMs());
Assert.assertEquals("Decommissioned Nodes", initialDecommissioned,
cm.getNumDecommisionedNMs());
-
- // Verify node in DECOMMISSIONING will be changed by status update
- // without running apps
- statusEvent = getMockRMNodeStatusEventWithoutRunningApps();
- node.handle(statusEvent);
- Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
- Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
- Assert.assertEquals("Decommissioning Nodes", initialDecommissioning - 1,
- cm.getNumDecommissioningNMs());
- Assert.assertEquals("Decommissioned Nodes", initialDecommissioned + 1,
- cm.getNumDecommisionedNMs());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 098ba54..aa5b336 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
@@ -87,6 +88,9 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
private final static File TEMP_DIR = new File(System.getProperty(
"test.build.data", "/tmp"), "decommision");
private final File hostFile = new File(TEMP_DIR + File.separator + "hostFile.txt");
+ private final File excludeHostFile = new File(TEMP_DIR + File.separator +
+ "excludeHostFile.txt");
+
private MockRM rm;
/**
@@ -222,6 +226,109 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
/**
+ * Graceful decommission node with no running application.
+ */
+ @Test
+ public void testGracefulDecommissionNoApp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+
+ writeToHostsFile("");
+ rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ MockNM nm3 = rm.registerNode("host3:4433", 5120);
+
+ int metricCount = ClusterMetrics.getMetrics().getNumDecommisionedNMs();
+ NodeHeartbeatResponse nodeHeartbeat1 = nm1.nodeHeartbeat(true);
+ NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true);
+ NodeHeartbeatResponse nodeHeartbeat3 = nm3.nodeHeartbeat(true);
+
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat1.getNodeAction()));
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat2.getNodeAction()));
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat3.getNodeAction()));
+
+ rm.waitForState(nm2.getNodeId(), NodeState.RUNNING);
+ rm.waitForState(nm3.getNodeId(), NodeState.RUNNING);
+
+ // Graceful decommission both host2 and host3.
+ writeToHostsFile("host2", "host3");
+ rm.getNodesListManager().refreshNodes(conf, true);
+
+ rm.waitForState(nm2.getNodeId(), NodeState.DECOMMISSIONING);
+ rm.waitForState(nm3.getNodeId(), NodeState.DECOMMISSIONING);
+
+ nodeHeartbeat1 = nm1.nodeHeartbeat(true);
+ nodeHeartbeat2 = nm2.nodeHeartbeat(true);
+ nodeHeartbeat3 = nm3.nodeHeartbeat(true);
+
+ checkDecommissionedNMCount(rm, metricCount + 2);
+ rm.waitForState(nm2.getNodeId(), NodeState.DECOMMISSIONED);
+ rm.waitForState(nm3.getNodeId(), NodeState.DECOMMISSIONED);
+
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat1.getNodeAction()));
+ Assert.assertEquals(NodeAction.SHUTDOWN, nodeHeartbeat2.getNodeAction());
+ Assert.assertEquals(NodeAction.SHUTDOWN, nodeHeartbeat3.getNodeAction());
+ }
+
+ /**
+ * Graceful decommission node with running application.
+ */
+ @Test
+ public void testGracefulDecommissionWithApp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+
+ writeToHostsFile("");
+ rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("host1:1234", 10240);
+ MockNM nm2 = rm.registerNode("host2:5678", 20480);
+ MockNM nm3 = rm.registerNode("host3:4433", 10240);
+ NodeId id1 = nm1.getNodeId();
+ NodeId id3 = nm3.getNodeId();
+ rm.waitForState(id1, NodeState.RUNNING);
+ rm.waitForState(id3, NodeState.RUNNING);
+
+ // Create an app and launch two containers on host1.
+ RMApp app = rm.submitApp(2000);
+ MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
+ ApplicationAttemptId aaid = app.getCurrentAppAttempt().getAppAttemptId();
+
+ // Graceful decommission host1 and host3
+ writeToHostsFile("host1", "host3");
+ rm.getNodesListManager().refreshNodes(conf, true);
+ rm.waitForState(id1, NodeState.DECOMMISSIONING);
+ rm.waitForState(id3, NodeState.DECOMMISSIONING);
+
+ // host1 should be DECOMMISSIONING due to running containers.
+ // host3 should become DECOMMISSIONED.
+ nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING);
+ nm3.nodeHeartbeat(true);
+ rm.waitForState(id1, NodeState.DECOMMISSIONING);
+ rm.waitForState(id3, NodeState.DECOMMISSIONED);
+ nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING);
+
+ // Complete containers on host1.
+ // Since the app is still RUNNING, expect NodeAction.NORMAL.
+ NodeHeartbeatResponse nodeHeartbeat1 =
+ nm1.nodeHeartbeat(aaid, 2, ContainerState.COMPLETE);
+ Assert.assertEquals(NodeAction.NORMAL, nodeHeartbeat1.getNodeAction());
+
+ // Finish the app and verified DECOMMISSIONED.
+ MockRM.finishAMAndVerifyAppState(app, rm, nm1, am);
+ rm.waitForState(app.getApplicationId(), RMAppState.FINISHED);
+ nodeHeartbeat1 = nm1.nodeHeartbeat(aaid, 2, ContainerState.COMPLETE);
+ Assert.assertEquals(NodeAction.SHUTDOWN, nodeHeartbeat1.getNodeAction());
+ rm.waitForState(id1, NodeState.DECOMMISSIONED);
+ }
+
+ /**
* Decommissioning using a post-configured include hosts file
*/
@Test
@@ -1225,19 +1332,17 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
MockNM nm1 = new MockNM("host1:1234", 5120, resourceTrackerService);
RegisterNodeManagerResponse response = nm1.registerNode();
Assert.assertEquals(NodeAction.NORMAL, response.getNodeAction());
+ int shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs();
writeToHostsFile("host2");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
hostFile.getAbsolutePath());
rm.getNodesListManager().refreshNodes(conf);
NodeHeartbeatResponse heartbeatResponse = nm1.nodeHeartbeat(true);
Assert.assertEquals(NodeAction.SHUTDOWN, heartbeatResponse.getNodeAction());
- int shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs();
- checkShutdownNMCount(rm, shutdownNMsCount);
checkDecommissionedNMCount(rm, decommisionedNMsCount);
request.setNodeId(nm1.getNodeId());
resourceTrackerService.unRegisterNodeManager(request);
- shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs();
- checkShutdownNMCount(rm, shutdownNMsCount);
+ checkShutdownNMCount(rm, ++shutdownNMsCount);
checkDecommissionedNMCount(rm, decommisionedNMsCount);
// 1. Register the Node Manager
@@ -1273,8 +1378,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
- File excludeHostFile =
- new File(TEMP_DIR + File.separator + "excludeHostFile.txt");
writeToHostsFile(excludeHostFile, "host1");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
excludeHostFile.getAbsolutePath());
@@ -1300,8 +1403,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
Assert.assertEquals("The inactiveRMNodes should contain an entry for the" +
"decommissioned node",
1, rm1.getRMContext().getInactiveRMNodes().size());
- excludeHostFile =
- new File(TEMP_DIR + File.separator + "excludeHostFile.txt");
writeToHostsFile(excludeHostFile, "");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
excludeHostFile.getAbsolutePath());
@@ -1331,8 +1432,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
//host3 will not register or heartbeat
- File excludeHostFile =
- new File(TEMP_DIR + File.separator + "excludeHostFile.txt");
writeToHostsFile(excludeHostFile, "host3", "host2");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
excludeHostFile.getAbsolutePath());
@@ -1364,14 +1463,12 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
MockNM nm2 = rm.registerNode("host2:5678", 10240);
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
- File excludeHostFile =
- new File(TEMP_DIR + File.separator + "excludeHostFile.txt");
writeToHostsFile(excludeHostFile, "host3", "host2");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
excludeHostFile.getAbsolutePath());
writeToHostsFile(hostFile, "host1", "host2");
writeToHostsFile(excludeHostFile, "host1");
- rm.getNodesListManager().refreshNodesGracefully(conf);
+ rm.getNodesListManager().refreshNodesGracefully(conf, null);
rm.drainEvents();
nm1.nodeHeartbeat(true);
rm.drainEvents();
@@ -1380,7 +1477,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
.getInactiveRMNodes().get(nm1.getNodeId()).getState() == NodeState
.DECOMMISSIONED);
writeToHostsFile(excludeHostFile, "");
- rm.getNodesListManager().refreshNodesGracefully(conf);
+ rm.getNodesListManager().refreshNodesGracefully(conf, null);
rm.drainEvents();
Assert.assertTrue("Node " + nm1.getNodeId().getHost() +
" should be Decommissioned", rm.getRMContext()
@@ -1390,7 +1487,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
/**
- * Remove a node from all lists and check if its forgotten
+ * Remove a node from all lists and check if its forgotten.
*/
@Test
public void testNodeRemovalNormally() throws Exception {
@@ -1411,7 +1508,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
public void refreshNodesOption(boolean doGraceful, Configuration conf)
throws Exception {
if (doGraceful) {
- rm.getNodesListManager().refreshNodesGracefully(conf);
+ rm.getNodesListManager().refreshNodesGracefully(conf, null);
} else {
rm.getNodesListManager().refreshNodes(conf);
}
@@ -1420,8 +1517,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
public void testNodeRemovalUtil(boolean doGraceful) throws Exception {
Configuration conf = new Configuration();
int timeoutValue = 500;
- File excludeHostFile = new File(TEMP_DIR + File.separator +
- "excludeHostFile.txt");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, "");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, "");
conf.setInt(YarnConfiguration.RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC,
@@ -1455,18 +1550,23 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
.getAbsolutePath());
refreshNodesOption(doGraceful, conf);
+ if (doGraceful) {
+ rm.waitForState(nm2.getNodeId(), NodeState.DECOMMISSIONING);
+ }
nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
rm.drainEvents();
Assert.assertTrue("Node should not be in active node list",
!rmContext.getRMNodes().containsKey(nm2.getNodeId()));
RMNode rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId());
Assert.assertEquals("Node should be in inactive node list",
- rmNode.getState(), NodeState.SHUTDOWN);
+ rmNode.getState(),
+ doGraceful? NodeState.DECOMMISSIONED : NodeState.SHUTDOWN);
Assert.assertEquals("Active nodes should be 2",
metrics.getNumActiveNMs(), 2);
- Assert.assertEquals("Shutdown nodes should be 1",
- metrics.getNumShutdownNMs(), 1);
+ Assert.assertEquals("Shutdown nodes should be expected",
+ metrics.getNumShutdownNMs(), doGraceful? 0 : 1);
int nodeRemovalTimeout =
conf.getInt(
@@ -1491,14 +1591,18 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm.drainEvents();
writeToHostsFile("host1", ip);
refreshNodesOption(doGraceful, conf);
+ rm.waitForState(nm2.getNodeId(),
+ doGraceful? NodeState.DECOMMISSIONING : NodeState.SHUTDOWN);
+ nm2.nodeHeartbeat(true);
rm.drainEvents();
rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId());
Assert.assertEquals("Node should be shutdown",
- rmNode.getState(), NodeState.SHUTDOWN);
+ rmNode.getState(),
+ doGraceful? NodeState.DECOMMISSIONED : NodeState.SHUTDOWN);
Assert.assertEquals("Active nodes should be 2",
metrics.getNumActiveNMs(), 2);
- Assert.assertEquals("Shutdown nodes should be 1",
- metrics.getNumShutdownNMs(), 1);
+ Assert.assertEquals("Shutdown nodes should be expected",
+ metrics.getNumShutdownNMs(), doGraceful? 0 : 1);
//add back the node before timer expires
latch.await(maxThreadSleeptime - 2000, TimeUnit.MILLISECONDS);
@@ -1542,6 +1646,20 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
//Test decommed/ing node that transitions to untracked,timer should remove
+ testNodeRemovalUtilDecomToUntracked(rmContext, conf, nm1, nm2, nm3,
+ maxThreadSleeptime, doGraceful);
+ rm.stop();
+ }
+
+ // A helper method used by testNodeRemovalUtil to avoid exceeding
+ // max allowed length.
+ private void testNodeRemovalUtilDecomToUntracked(
+ RMContext rmContext, Configuration conf,
+ MockNM nm1, MockNM nm2, MockNM nm3,
+ long maxThreadSleeptime, boolean doGraceful) throws Exception {
+ ClusterMetrics metrics = ClusterMetrics.getMetrics();
+ String ip = NetUtils.normalizeHostName("localhost");
+ CountDownLatch latch = new CountDownLatch(1);
writeToHostsFile("host1", ip, "host2");
writeToHostsFile(excludeHostFile, "host2");
refreshNodesOption(doGraceful, conf);
@@ -1549,7 +1667,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
//nm2.nodeHeartbeat(true);
nm3.nodeHeartbeat(true);
latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS);
- rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
+ RMNode rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
rmContext.getInactiveRMNodes().get(nm2.getNodeId());
Assert.assertNotEquals("Timer for this node was not canceled!",
rmNode, null);
@@ -1560,6 +1678,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
writeToHostsFile("host1", ip);
writeToHostsFile(excludeHostFile, "");
refreshNodesOption(doGraceful, conf);
+ nm2.nodeHeartbeat(true);
latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS);
rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
rmContext.getInactiveRMNodes().get(nm2.getNodeId());
@@ -1571,16 +1690,12 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
metrics.getNumShutdownNMs(), 0);
Assert.assertEquals("Active nodes should be 2",
metrics.getNumActiveNMs(), 2);
-
- rm.stop();
}
private void testNodeRemovalUtilLost(boolean doGraceful) throws Exception {
Configuration conf = new Configuration();
conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 2000);
int timeoutValue = 500;
- File excludeHostFile = new File(TEMP_DIR + File.separator +
- "excludeHostFile.txt");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
hostFile.getAbsolutePath());
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
@@ -1613,7 +1728,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
Assert.assertEquals("All 3 nodes should be active",
metrics.getNumActiveNMs(), 3);
int waitCount = 0;
- while(waitCount ++<20){
+ while(waitCount++ < 20){
synchronized (this) {
wait(200);
}
@@ -1665,8 +1780,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
throws Exception {
Configuration conf = new Configuration();
int timeoutValue = 500;
- File excludeHostFile = new File(TEMP_DIR + File.separator +
- "excludeHostFile.txt");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
hostFile.getAbsolutePath());
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
@@ -1737,8 +1850,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
throws Exception {
Configuration conf = new Configuration();
int timeoutValue = 500;
- File excludeHostFile = new File(TEMP_DIR + File.separator +
- "excludeHostFile.txt");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
hostFile.getAbsolutePath());
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
@@ -1782,15 +1893,19 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
nm2.nodeHeartbeat(false);
nm3.nodeHeartbeat(true);
rm.drainEvents();
- Assert.assertNotEquals("host2 should be a shutdown NM!",
- rmContext.getInactiveRMNodes().get(nm2.getNodeId()), null);
- Assert.assertEquals("host2 should be a shutdown NM!",
- rmContext.getInactiveRMNodes().get(nm2.getNodeId()).getState(),
- NodeState.SHUTDOWN);
+ if (!doGraceful) {
+ Assert.assertNotEquals("host2 should be a shutdown NM!",
+ rmContext.getInactiveRMNodes().get(nm2.getNodeId()), null);
+ Assert.assertEquals("host2 should be a shutdown NM!",
+ rmContext.getInactiveRMNodes().get(nm2.getNodeId()).getState(),
+ NodeState.SHUTDOWN);
+ }
Assert.assertEquals("There should be 2 Active NM!",
clusterMetrics.getNumActiveNMs(), 2);
- Assert.assertEquals("There should be 1 Shutdown NM!",
- clusterMetrics.getNumShutdownNMs(), 1);
+ if (!doGraceful) {
+ Assert.assertEquals("There should be 1 Shutdown NM!",
+ clusterMetrics.getNumShutdownNMs(), 1);
+ }
Assert.assertEquals("There should be 0 Unhealthy NM!",
clusterMetrics.getUnhealthyNMs(), 0);
int nodeRemovalTimeout =
@@ -1818,7 +1933,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
private void writeToHostsFile(String... hosts) throws IOException {
- writeToHostsFile(hostFile, hosts);
+ writeToHostsFile(hostFile, hosts);
}
private void writeToHostsFile(File file, String... hosts)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
index 2c926d9..e7c7e51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
@@ -210,8 +210,6 @@ public class TestNMReconnect extends ParameterizedSchedulerTestBase {
nm1.registerNode();
rm.waitForState(nm1.getNodeId(), NodeState.RUNNING);
- rm.getRMContext().getNodesListManager().getHostsReader().
- getExcludedHosts().add("127.0.0.1");
rm.getRMContext().getDispatcher().getEventHandler().handle(
new RMNodeEvent(nm1.getNodeId(),
RMNodeEventType.GRACEFUL_DECOMMISSION));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 40704f0..2c38967 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -243,7 +243,7 @@ Usage:
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| -refreshQueues | Reload the queues' acls, states and scheduler specific properties. ResourceManager will reload the mapred-queues configuration file. |
-| -refreshNodes | Refresh the hosts information at the ResourceManager. |
+| -refreshNodes [-g|graceful [timeout in seconds] -client|server] | Refresh the hosts information at the ResourceManager. -g option indicates graceful decommission of excluded hosts, in which case, the optional timeout indicates maximal time in seconds ResourceManager should wait before forcefully mark the node as decommissioned. |
| -refreshNodesResources | Refresh resources of NodeManagers at the ResourceManager. |
| -refreshSuperUserGroupsConfiguration | Refresh superuser proxy groups mappings. |
| -refreshUserToGroupsMappings | Refresh user-to-groups mappings. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HADOOP-13428. Fixed hadoop-common
build files to generate jdiff by working around a jdiff-bug. Contributed by
Wangda Tan.
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99603e90/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index c28a05c..54d1cdd 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -34,6 +34,7 @@
<kdc.resource.dir>src/test/resources/kdc</kdc.resource.dir>
<hadoop.component>common</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
+ <is.hadoop.common.component>true</is.hadoop.common.component>
<wsce.config.dir>../etc/hadoop</wsce.config.dir>
<wsce.config.file>wsce-site.xml</wsce.config.file>
</properties>
@@ -512,6 +513,7 @@
<exclude>src/test/resources/test.har/_masterindex</exclude>
<exclude>src/test/resources/test.har/part-0</exclude>
<exclude>src/test/resources/javakeystoreprovider.password</exclude>
+ <exclude>dev-support/jdiff-workaround.patch</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99603e90/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 0357269..0ee9895 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -160,7 +160,7 @@
<activeByDefault>false</activeByDefault>
</activation>
<properties>
- <jdiff.stable.api>2.6.0</jdiff.stable.api>
+ <jdiff.stable.api>2.7.2</jdiff.stable.api>
<jdiff.stability>-unstable</jdiff.stability>
<!-- Commented out for HADOOP-11776 -->
<!-- Uncomment param name="${jdiff.compatibility}" in javadoc doclet if compatibility is not empty -->
@@ -173,7 +173,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
- <execution>
+ <execution>
<goals>
<goal>javadoc</goal>
</goals>
@@ -241,6 +241,26 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
+
+ <!-- Pre site -->
+ <execution>
+ <id>pre-site</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target if="is.hadoop.common.component">
+ <!-- HADOOP-13428: This is for workaround issues of JDiff
+ - https://sourceforge.net/p/javadiff/bugs/19/ -->
+ <exec executable="patch" input="dev-support/jdiff-workaround.patch" dir="${basedir}">
+ <arg value="-p3"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+
+
<execution>
<id>site</id>
<phase>prepare-package</phase>
@@ -249,7 +269,6 @@
</goals>
<configuration>
<target if="is.hadoop.component">
-
<mkdir dir="${project.build.directory}/docs-src"/>
<!-- Docs -->
@@ -302,6 +321,25 @@
</target>
</configuration>
</execution>
+
+ <!-- post site phase -->
+ <execution>
+ <id>post-site</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target if="is.hadoop.common.component">
+ <!-- HADOOP-13428: Revert temporary patch-->
+ <exec executable="patch" input="dev-support/jdiff-workaround.patch" dir="${basedir}">
+ <arg value="-p3"/>
+ <arg value="-R"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+
</executions>
</plugin>
</plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDFS-10783. The option '-maxSize'
and '-step' fail in OfflineImageViewer. Contributed by Yiqun Lin.
Posted by cd...@apache.org.
HDFS-10783. The option '-maxSize' and '-step' fail in OfflineImageViewer. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e90f3359
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e90f3359
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e90f3359
Branch: refs/heads/HDFS-9806
Commit: e90f3359de299ef5e3a54ca71070e3dfe1dbb98c
Parents: 0d5997d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Aug 23 19:56:27 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Aug 23 19:57:23 2016 +0900
----------------------------------------------------------------------
.../offlineImageViewer/OfflineImageViewer.java | 2 ++
.../TestOfflineImageViewer.java | 19 +++++++++++++++++++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e90f3359/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
index 7f81ba8..770cde1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
@@ -186,6 +186,8 @@ public class OfflineImageViewer {
options.addOption("p", "processor", true, "");
options.addOption("h", "help", false, "");
+ options.addOption("maxSize", true, "");
+ options.addOption("step", true, "");
options.addOption("skipBlocks", false, "");
options.addOption("printToScreen", false, "");
options.addOption("delimiter", true, "");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e90f3359/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index b9aa7f3..a7c30ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -601,4 +601,23 @@ public class TestOfflineImageViewer {
"FileDistribution", "-maxSize", "23", "-step", "4"});
assertEquals(0, status);
}
+
+ @Test
+ public void testOfflineImageViewerMaxSizeAndStepOptions() throws Exception {
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ final PrintStream out = new PrintStream(bytes);
+ final PrintStream oldOut = System.out;
+ try {
+ System.setOut(out);
+ // Add the -h option to make the test only for option parsing,
+ // and don't need to do the following operations.
+ OfflineImageViewer.main(new String[] {"-i", "-", "-o", "-", "-p",
+ "FileDistribution", "-maxSize", "512", "-step", "8", "-h"});
+ Assert.assertFalse(bytes.toString().contains(
+ "Error parsing command-line options: "));
+ } finally {
+ System.setOut(oldOut);
+ IOUtils.closeStream(out);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HADOOP-13428. Fixed hadoop-common
build files to generate jdiff by working around a jdiff-bug. Contributed by
Wangda Tan.
Posted by cd...@apache.org.
HADOOP-13428. Fixed hadoop-common build files to generate jdiff by working around a jdiff-bug. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99603e90
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99603e90
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99603e90
Branch: refs/heads/HDFS-9806
Commit: 99603e902244f17b04cfd55122f47355d070b588
Parents: 2da32a6
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Fri Aug 19 19:08:53 2016 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Fri Aug 19 19:08:53 2016 -0700
----------------------------------------------------------------------
.../dev-support/jdiff-workaround.patch | 98 +
.../jdiff/Apache_Hadoop_Common_2.7.2.xml | 46648 +++++++++++++++++
hadoop-common-project/hadoop-common/pom.xml | 2 +
hadoop-project-dist/pom.xml | 44 +-
4 files changed, 46789 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99603e90/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
new file mode 100644
index 0000000..8f87d40
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
@@ -0,0 +1,98 @@
+diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
+index a277abd..ed7c709 100644
+--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
+@@ -43,18 +43,6 @@
+ public abstract MetricsSystem init(String prefix);
+
+ /**
+- * Register a metrics source
+- * @param <T> the actual type of the source object
+- * @param source object to register
+- * @param name of the source. Must be unique or null (then extracted from
+- * the annotations of the source object.)
+- * @param desc the description of the source (or null. See above.)
+- * @return the source object
+- * @exception MetricsException
+- */
+- public abstract <T> T register(String name, String desc, T source);
+-
+- /**
+ * Unregister a metrics source
+ * @param name of the source. This is the name you use to call register()
+ */
+@@ -77,18 +65,19 @@
+ */
+ @InterfaceAudience.Private
+ public abstract MetricsSource getSource(String name);
++
+
+ /**
+- * Register a metrics sink
+- * @param <T> the type of the sink
+- * @param sink to register
+- * @param name of the sink. Must be unique.
+- * @param desc the description of the sink
+- * @return the sink
++ * Register a metrics source
++ * @param <T> the actual type of the source object
++ * @param source object to register
++ * @param name of the source. Must be unique or null (then extracted from
++ * the annotations of the source object.)
++ * @param desc the description of the source (or null. See above.)
++ * @return the source object
+ * @exception MetricsException
+ */
+- public abstract <T extends MetricsSink>
+- T register(String name, String desc, T sink);
++ public abstract <T> T register(String name, String desc, T source);
+
+ /**
+ * Register a callback interface for JMX events
+diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+index 6986edb..eeea81f 100644
+--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+@@ -270,27 +270,6 @@ void registerSource(String name, String desc, MetricsSource source) {
+ LOG.debug("Registered source "+ name);
+ }
+
+- @Override public synchronized <T extends MetricsSink>
+- T register(final String name, final String description, final T sink) {
+- LOG.debug(name +", "+ description);
+- if (allSinks.containsKey(name)) {
+- LOG.warn("Sink "+ name +" already exists!");
+- return sink;
+- }
+- allSinks.put(name, sink);
+- if (config != null) {
+- registerSink(name, description, sink);
+- }
+- // We want to re-register the sink to pick up new config
+- // when the metrics system restarts.
+- register(name, new AbstractCallback() {
+- @Override public void postStart() {
+- register(name, description, sink);
+- }
+- });
+- return sink;
+- }
+-
+ synchronized void registerSink(String name, String desc, MetricsSink sink) {
+ checkNotNull(config, "config");
+ MetricsConfig conf = sinkConfigs.get(name);
+diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
+index c19d238..f8412f1 100644
+--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
++++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
+@@ -130,8 +130,8 @@ public void testTagsForPrefix() throws Exception {
+ GangliaMetricsTestHelper.setDatagramSocket(gsink31, mockds31);
+
+ // register the sinks
+- ms.register("gsink30", "gsink30 desc", gsink30);
+- ms.register("gsink31", "gsink31 desc", gsink31);
++ //ms.register("gsink30", "gsink30 desc", gsink30);
++ //ms.register("gsink31", "gsink31 desc", gsink31);
+ ms.publishMetricsNow(); // publish the metrics
+
+ ms.stop();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HADOOP-13513. Java 1.7 support for
org.apache.hadoop.fs.azure testcases. Contributed by Tibor Kiss.
Posted by cd...@apache.org.
HADOOP-13513. Java 1.7 support for org.apache.hadoop.fs.azure testcases. Contributed by Tibor Kiss.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae4db254
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae4db254
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae4db254
Branch: refs/heads/HDFS-9806
Commit: ae4db2544346370404826d5b55b2678f5f92fe1f
Parents: 0da69c3
Author: Chris Nauroth <cn...@apache.org>
Authored: Thu Aug 18 10:44:36 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Thu Aug 18 10:44:36 2016 -0700
----------------------------------------------------------------------
.../test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae4db254/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
index f0df38b..58d278a 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
@@ -35,7 +35,7 @@ import org.slf4j.LoggerFactory;
* to run those tests. The returned account might integrate with Azure Storage
* directly or it might be a mock implementation.
*/
-abstract class AbstractWasbTestBase {
+public abstract class AbstractWasbTestBase {
protected static final Logger LOG =
LoggerFactory.getLogger(AbstractWasbTestBase.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: Revert "YARN-5533. JMX AM Used
metrics for queue wrong when app submited to nodelabel partition (Bibin A
Chundatt via Varun Saxena)"
Posted by cd...@apache.org.
Revert "YARN-5533. JMX AM Used metrics for queue wrong when app submited to nodelabel partition (Bibin A Chundatt via Varun Saxena)"
This reverts commit 59557e85a40fb91c7106e8ff3bfe958ffa244b29.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aed3741
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aed3741
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aed3741
Branch: refs/heads/HDFS-9806
Commit: 8aed374182f22c31e886a9a602fb907bbccc4309
Parents: 59557e8
Author: Varun Saxena <va...@apache.org>
Authored: Fri Aug 19 16:14:16 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Aug 19 16:14:16 2016 +0530
----------------------------------------------------------------------
.../resourcemanager/scheduler/capacity/LeafQueue.java | 3 +--
.../capacity/TestNodeLabelContainerAllocation.java | 9 ---------
2 files changed, 1 insertion(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aed3741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 636762f..6bbe85e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -774,8 +774,7 @@ public class LeafQueue extends AbstractCSQueue {
application.getAMResource(partitionName));
user.getResourceUsage().decAMUsed(partitionName,
application.getAMResource(partitionName));
- metrics.decAMUsed(application.getUser(),
- application.getAMResource(partitionName));
+ metrics.decAMUsed(application.getUser(), application.getAMResource());
}
applicationAttemptMap.remove(application.getApplicationAttemptId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aed3741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 251e4dc..9070577 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
@@ -1960,14 +1959,6 @@ public class TestNodeLabelContainerAllocation {
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
assertEquals(0 * GB, leafQueue.getMetrics().getAvailableMB());
assertEquals(5 * GB, leafQueue.getMetrics().getAllocatedMB());
-
- // Kill all apps in queue a
- cs.killAllAppsInQueue("a");
- rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
- rm1.waitForAppRemovedFromScheduler(app1.getApplicationId());
-
- assertEquals(0 * GB, leafQueue.getMetrics().getUsedAMResourceMB());
- assertEquals(0 * GB, leafQueue.getMetrics().getUsedAMResourceVCores());
rm1.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: MAPREDUCE-6310. Add jdiff support
to MapReduce. (Li Lu/vinodkv via wangda)
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.7.2.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.7.2.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.7.2.xml
new file mode 100644
index 0000000..77074d3
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.7.2.xml
@@ -0,0 +1,31129 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon Jun 13 20:32:44 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="hadoop-mapreduce-client-core 2.7.2"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-annotations.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/jdiff.jar -verbose -classpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/classes:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-2.7.2.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-2.7.2.jar:/Users/vinodkv/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/
Users/vinodkv/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/vinodkv/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/vinodkv/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/vinodkv/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/vinodkv/.m2/repository/org/co
dehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/vinodkv/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/vinodkv/.m2/repository/com/google/inject/guice/3.0/guice-3.0.jar:/Users/vinodkv/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/Users/vinodkv/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/vinodkv/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/vinodkv/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/Users/vinodkv/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/vinodkv/
.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/vinodkv/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar:/Users/vinodkv/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/vinodkv/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/vinodkv/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/vinodkv/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-common/target/hadoop-common-2.7.2.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/vinodkv/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/vinodkv/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/vinodkv/.m2/repository/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/Users/vinodkv/.
m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/vinodkv/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/vinodkv/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/vinodkv/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/vinodkv/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.7.2.jar:/Users/vinodkv/.m2/repository/org/apache/d
irectory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar:/Users/vinodkv/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-recipes/2.7.1/curator-recipes-2.7.1.jar:/Users/vinodkv/.m2/repository/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar:/Users/vinodkv/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-
api-1.7.10.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.7.2.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home/lib/tools.jar:/Users/vinodkv/.m2/repository/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar:/Users/vinodkv/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/vinodkv/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/vinodkv/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/vinodkv/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/vinodkv/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/vinodkv/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/vinodkv/.m2/repository/commons-collections/commons-collections/3.2.2/commons-coll
ections-3.2.2.jar -sourcepath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java -apidir /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/site/jdiff/xml -apiname hadoop-mapreduce-client-core 2.7.2 -->
+<package name="org.apache.hadoop.filecache">
+ <!-- start class org.apache.hadoop.filecache.DistributedCache -->
+ <class name="DistributedCache" extends="org.apache.hadoop.mapreduce.filecache.DistributedCache"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="DistributedCache"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="addLocalArchives"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="str" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a archive that has been localized to the conf. Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+ </doc>
+ </method>
+ <method name="addLocalFiles"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="str" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a file that has been localized to the conf.. Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+ </doc>
+ </method>
+ <method name="createAllSymlink"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="Internal to MapReduce framework. Use DistributedCacheManager
+ instead.">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="jobCacheDir" type="java.io.File"/>
+ <param name="workDir" type="java.io.File"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[This method create symlinks for all files in a given dir in another
+ directory. Currently symlinks cannot be disabled. This is a NO-OP.
+
+ @param conf the configuration
+ @param jobCacheDir the target directory for creating symlinks
+ @param workDir the directory in which the symlinks are created
+ @throws IOException
+ @deprecated Internal to MapReduce framework. Use DistributedCacheManager
+ instead.]]>
+ </doc>
+ </method>
+ <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="cache" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns {@link FileStatus} of a given cache file on hdfs. Internal to
+ MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return <code>FileStatus</code> of a given cache file on hdfs
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getTimestamp" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="cache" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns mtime of a given cache file on hdfs. Internal to MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return mtime of a given cache file on hdfs
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="setArchiveTimestamps"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="timestamps" type="java.lang.String"/>
+ <doc>
+ <![CDATA[This is to check the timestamp of the archives to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of archives.
+ The order should be the same as the order in which the archives are added.]]>
+ </doc>
+ </method>
+ <method name="setFileTimestamps"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="timestamps" type="java.lang.String"/>
+ <doc>
+ <![CDATA[This is to check the timestamp of the files to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of files.
+ The order should be the same as the order in which the files are added.]]>
+ </doc>
+ </method>
+ <method name="setLocalArchives"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="str" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the conf to contain the location for localized archives. Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+ </doc>
+ </method>
+ <method name="setLocalFiles"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="str" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the conf to contain the location for localized files. Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+ </doc>
+ </method>
+ <field name="CACHE_FILES_SIZES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_FILES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES_SIZES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_ARCHIVES_SIZES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_ARCHIVES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_SIZES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_ARCHIVES_TIMESTAMPS" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_ARCHIVES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_TIMESTAMPS}]]>
+ </doc>
+ </field>
+ <field name="CACHE_FILES_TIMESTAMPS" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_FILES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILE_TIMESTAMPS}]]>
+ </doc>
+ </field>
+ <field name="CACHE_ARCHIVES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_ARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_FILES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_FILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_LOCALARCHIVES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_LOCALARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALARCHIVES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_LOCALFILES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_LOCALFILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALFILES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_SYMLINK" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_SYMLINK} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_SYMLINK}]]>
+ </doc>
+ </field>
+ <doc>
+ <![CDATA[Distribute application-specific large, read-only files efficiently.
+
+ <p><code>DistributedCache</code> is a facility provided by the Map-Reduce
+ framework to cache files (text, archives, jars etc.) needed by applications.
+ </p>
+
+ <p>Applications specify the files, via urls (hdfs:// or http://) to be cached
+ via the {@link org.apache.hadoop.mapred.JobConf}. The
+ <code>DistributedCache</code> assumes that the files specified via urls are
+ already present on the {@link FileSystem} at the path specified by the url
+ and are accessible by every machine in the cluster.</p>
+
+ <p>The framework will copy the necessary files on to the slave node before
+ any tasks for the job are executed on that node. Its efficiency stems from
+ the fact that the files are only copied once per job and the ability to
+ cache archives which are un-archived on the slaves.</p>
+
+ <p><code>DistributedCache</code> can be used to distribute simple, read-only
+ data/text files and/or more complex types such as archives, jars etc.
+ Archives (zip, tar and tgz/tar.gz files) are un-archived at the slave nodes.
+ Jars may be optionally added to the classpath of the tasks, a rudimentary
+ software distribution mechanism. Files have execution permissions.
+ In older version of Hadoop Map/Reduce users could optionally ask for symlinks
+ to be created in the working directory of the child task. In the current
+ version symlinks are always created. If the URL does not have a fragment
+ the name of the file or directory will be used. If multiple files or
+ directories map to the same link name, the last one added, will be used. All
+ others will not even be downloaded.</p>
+
+ <p><code>DistributedCache</code> tracks modification timestamps of the cache
+ files. Clearly the cache files should not be modified by the application
+ or externally while the job is executing.</p>
+
+ <p>Here is an illustrative example on how to use the
+ <code>DistributedCache</code>:</p>
+ <p><blockquote><pre>
+ // Setting up the cache for the application
+
+ 1. Copy the requisite files to the <code>FileSystem</code>:
+
+ $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat
+ $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip
+ $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
+ $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
+ $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
+ $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
+
+ 2. Setup the application's <code>JobConf</code>:
+
+ JobConf job = new JobConf();
+ DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"),
+ job);
+ DistributedCache.addCacheArchive(new URI("/myapp/map.zip", job);
+ DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
+ DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar", job);
+ DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz", job);
+ DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz", job);
+
+ 3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
+ or {@link org.apache.hadoop.mapred.Reducer}:
+
+ public static class MapClass extends MapReduceBase
+ implements Mapper<K, V, K, V> {
+
+ private Path[] localArchives;
+ private Path[] localFiles;
+
+ public void configure(JobConf job) {
+ // Get the cached archives/files
+ File f = new File("./map.zip/some/file/in/zip.txt");
+ }
+
+ public void map(K key, V value,
+ OutputCollector<K, V> output, Reporter reporter)
+ throws IOException {
+ // Use data from the cached archives/files here
+ // ...
+ // ...
+ output.collect(k, v);
+ }
+ }
+
+ </pre></blockquote>
+
+ It is also very common to use the DistributedCache by using
+ {@link org.apache.hadoop.util.GenericOptionsParser}.
+
+ This class includes methods that should be used by users
+ (specifically those mentioned in the example above, as well
+ as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}),
+ as well as methods intended for use by the MapReduce framework
+ (e.g., {@link org.apache.hadoop.mapred.JobClient}).
+
+ @see org.apache.hadoop.mapred.JobConf
+ @see org.apache.hadoop.mapred.JobClient
+ @see org.apache.hadoop.mapreduce.Job]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.filecache.DistributedCache -->
+</package>
+<package name="org.apache.hadoop.mapred">
+ <!-- start class org.apache.hadoop.mapred.ClusterStatus -->
+ <class name="ClusterStatus" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.io.Writable"/>
+ <method name="getTaskTrackers" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of task trackers in the cluster.
+
+ @return the number of task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getActiveTrackerNames" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the names of task trackers in the cluster.
+
+ @return the active task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getBlacklistedTrackerNames" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the names of task trackers in the cluster.
+
+ @return the blacklisted task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getGraylistedTrackerNames" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the names of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return an empty graylisted task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getGraylistedTrackers" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return 0 graylisted task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getBlacklistedTrackers" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of blacklisted task trackers in the cluster.
+
+ @return the number of blacklisted task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getNumExcludedNodes" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of excluded hosts in the cluster.
+ @return the number of excluded hosts in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getTTExpiryInterval" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the tasktracker expiry interval for the cluster
+ @return the expiry interval in msec]]>
+ </doc>
+ </method>
+ <method name="getMapTasks" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of currently running map tasks in the cluster.
+
+ @return the number of currently running map tasks in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getReduceTasks" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of currently running reduce tasks in the cluster.
+
+ @return the number of currently running reduce tasks in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getMaxMapTasks" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the maximum capacity for running map tasks in the cluster.
+
+ @return the maximum capacity for running map tasks in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getMaxReduceTasks" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the maximum capacity for running reduce tasks in the cluster.
+
+ @return the maximum capacity for running reduce tasks in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the JobTracker's status.
+
+ @return {@link JobTrackerStatus} of the JobTracker]]>
+ </doc>
+ </method>
+ <method name="getMaxMemory" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+ </doc>
+ </method>
+ <method name="getUsedMemory" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+ </doc>
+ </method>
+ <method name="getBlackListedTrackersInfo" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets the list of blacklisted trackers along with reasons for blacklisting.
+
+ @return the collection of {@link BlackListInfo} objects.]]>
+ </doc>
+ </method>
+ <method name="getJobTrackerState" return="org.apache.hadoop.mapred.JobTracker.State"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the current state of the <code>JobTracker</code>,
+ as {@link JobTracker.State}
+
+ {@link JobTracker.State} should no longer be used on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return the invalid state of the <code>JobTracker</code>.]]>
+ </doc>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <field name="UNINITIALIZED_MEMORY_VALUE" type="long"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[Status information on the current state of the Map-Reduce cluster.
+
+ <p><code>ClusterStatus</code> provides clients with information such as:
+ <ol>
+ <li>
+ Size of the cluster.
+ </li>
+ <li>
+ Name of the trackers.
+ </li>
+ <li>
+ Task capacity of the cluster.
+ </li>
+ <li>
+ The number of currently running map and reduce tasks.
+ </li>
+ <li>
+ State of the <code>JobTracker</code>.
+ </li>
+ <li>
+ Details regarding black listed trackers.
+ </li>
+ </ol>
+
+ <p>Clients can query for the latest <code>ClusterStatus</code>, via
+ {@link JobClient#getClusterStatus()}.</p>
+
+ @see JobClient]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ClusterStatus -->
+ <!-- start class org.apache.hadoop.mapred.ClusterStatus.BlackListInfo -->
+ <class name="ClusterStatus.BlackListInfo" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.io.Writable"/>
+ <method name="getTrackerName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets the blacklisted tasktracker's name.
+
+ @return tracker's name.]]>
+ </doc>
+ </method>
+ <method name="getReasonForBlackListing" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets the reason for which the tasktracker was blacklisted.
+
+ @return reason which tracker was blacklisted]]>
+ </doc>
+ </method>
+ <method name="getBlackListReport" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets a descriptive report about why the tasktracker was blacklisted.
+
+ @return report describing why the tasktracker was blacklisted.]]>
+ </doc>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Class which encapsulates information about a blacklisted tasktracker.
+
+ The information includes the tasktracker's name and reasons for
+ getting blacklisted. The toString method of the class will print
+ the information in a whitespace separated fashion to enable parsing.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ClusterStatus.BlackListInfo -->
+ <!-- start class org.apache.hadoop.mapred.Counters -->
+ <class name="Counters" extends="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Counters"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="Counters" type="org.apache.hadoop.mapreduce.Counters"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getGroup" return="org.apache.hadoop.mapred.Counters.Group"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="groupName" type="java.lang.String"/>
+ </method>
+ <method name="getGroupNames" return="java.util.Collection"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="makeCompactString" return="java.lang.String"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="group" type="java.lang.String"/>
+ <param name="name" type="java.lang.String"/>
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="use {@link #findCounter(String, String)} instead">
+ <param name="group" type="java.lang.String"/>
+ <param name="id" type="int"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Find a counter by using strings
+ @param group the name of the group
+ @param id the id of the counter within the group (0 to N-1)
+ @param name the internal name of the counter
+ @return the counter for that name
+ @deprecated use {@link #findCounter(String, String)} instead]]>
+ </doc>
+ </method>
+ <method name="incrCounter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.Enum"/>
+ <param name="amount" type="long"/>
+ <doc>
+ <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param key identifies a counter
+ @param amount amount by which counter is to be incremented]]>
+ </doc>
+ </method>
+ <method name="incrCounter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="group" type="java.lang.String"/>
+ <param name="counter" type="java.lang.String"/>
+ <param name="amount" type="long"/>
+ <doc>
+ <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param group the name of the group
+ @param counter the internal name of the counter
+ @param amount amount by which counter is to be incremented]]>
+ </doc>
+ </method>
+ <method name="getCounter" return="long"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.Enum"/>
+ <doc>
+ <![CDATA[Returns current value of the specified counter, or 0 if the counter
+ does not exist.
+ @param key the counter enum to lookup
+ @return the counter value or 0 if counter not found]]>
+ </doc>
+ </method>
+ <method name="incrAllCounters"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="other" type="org.apache.hadoop.mapred.Counters"/>
+ <doc>
+ <![CDATA[Increments multiple counters by their amounts in another Counters
+ instance.
+ @param other the other Counters instance]]>
+ </doc>
+ </method>
+ <method name="size" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="use {@link #countCounters()} instead">
+ <doc>
+ <![CDATA[@return the total number of counters
+ @deprecated use {@link #countCounters()} instead]]>
+ </doc>
+ </method>
+ <method name="sum" return="org.apache.hadoop.mapred.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="a" type="org.apache.hadoop.mapred.Counters"/>
+ <param name="b" type="org.apache.hadoop.mapred.Counters"/>
+ <doc>
+ <![CDATA[Convenience method for computing the sum of two sets of counters.
+ @param a the first counters
+ @param b the second counters
+ @return a new summed counters object]]>
+ </doc>
+ </method>
+ <method name="log"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="log" type="org.apache.commons.logging.Log"/>
+ <doc>
+ <![CDATA[Logs the current counter values.
+ @param log The log to use.]]>
+ </doc>
+ </method>
+ <method name="makeEscapedCompactString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Represent the counter in a textual format that can be converted back to
+ its object form
+ @return the string in the following format
+ {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*]]>
+ </doc>
+ </method>
+ <method name="fromEscapedCompactString" return="org.apache.hadoop.mapred.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="compactString" type="java.lang.String"/>
+ <exception name="ParseException" type="java.text.ParseException"/>
+ <doc>
+ <![CDATA[Convert a stringified (by {@link #makeEscapedCompactString()} counter
+ representation into a counter object.
+ @param compactString to parse
+ @return a new counters object
+ @throws ParseException]]>
+ </doc>
+ </method>
+ <field name="MAX_COUNTER_LIMIT" type="int"
+ transient="false" volatile="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="MAX_GROUP_LIMIT" type="int"
+ transient="false" volatile="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[A set of named counters.
+
+ <p><code>Counters</code> represent global counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ any {@link Enum} type.</p>
+
+ <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
+ counters from a particular <code>Enum</code> class.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.Counters -->
+ <!-- start class org.apache.hadoop.mapred.Counters.Counter -->
+ <class name="Counters.Counter" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.Counter"/>
+ <constructor name="Counters.Counter"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setDisplayName"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="displayName" type="java.lang.String"/>
+ </method>
+ <method name="getName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDisplayName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getValue" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setValue"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="value" type="long"/>
+ </method>
+ <method name="increment"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="incr" type="long"/>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="makeEscapedCompactString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Returns the compact stringified version of the counter in the format
+ [(actual-name)(display-name)(value)]
+ @return the stringified result]]>
+ </doc>
+ </method>
+ <method name="contentEquals" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="deprecated, no comment">
+ <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+ <doc>
+ <![CDATA[Checks for (content) equality of two (basic) counters
+ @param counter to compare
+ @return true if content equals
+ @deprecated]]>
+ </doc>
+ </method>
+ <method name="getCounter" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return the value of the counter]]>
+ </doc>
+ </method>
+ <method name="getUnderlyingCounter" return="org.apache.hadoop.mapreduce.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="genericRight" type="java.lang.Object"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[A counter record, comprising its name and value.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.Counters.Counter -->
+ <!-- start class org.apache.hadoop.mapred.Counters.CountersExceededException -->
+ <class name="Counters.CountersExceededException" extends="java.lang.RuntimeException"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Counters.CountersExceededException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="Counters.CountersExceededException" type="org.apache.hadoop.mapred.Counters.CountersExceededException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <doc>
+ <![CDATA[Counter exception thrown when the number of counters exceed the limit]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.Counters.CountersExceededException -->
+ <!-- start class org.apache.hadoop.mapred.Counters.Group -->
+ <class name="Counters.Group" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+ <constructor name="Counters.Group"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getCounter" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counterName" type="java.lang.String"/>
+ <doc>
+ <![CDATA[@param counterName the name of the counter
+ @return the value of the specified counter, or 0 if the counter does
+ not exist.]]>
+ </doc>
+ </method>
+ <method name="makeEscapedCompactString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return the compact stringified version of the group in the format
+ {(actual-name)(display-name)(value)[][][]} where [] are compact strings
+ for the counters within.]]>
+ </doc>
+ </method>
+ <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="use {@link #findCounter(String)} instead">
+ <param name="id" type="int"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the counter for the given id and create it if it doesn't exist.
+ @param id the numeric id of the counter within the group
+ @param name the internal counter name
+ @return the counter
+ @deprecated use {@link #findCounter(String)} instead]]>
+ </doc>
+ </method>
+ <method name="getCounterForName" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the counter for the given name and create it if it doesn't exist.
+ @param name the internal counter name
+ @return the counter]]>
+ </doc>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="iterator" return="java.util.Iterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDisplayName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setDisplayName"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="displayName" type="java.lang.String"/>
+ </method>
+ <method name="addCounter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+ </method>
+ <method name="addCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="displayName" type="java.lang.String"/>
+ <param name="value" type="long"/>
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counterName" type="java.lang.String"/>
+ <param name="displayName" type="java.lang.String"/>
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counterName" type="java.lang.String"/>
+ <param name="create" type="boolean"/>
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counterName" type="java.lang.String"/>
+ </method>
+ <method name="size" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="incrAllCounters"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="rightGroup" type="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+ </method>
+ <method name="getUnderlyingGroup" return="org.apache.hadoop.mapreduce.counters.CounterGroupBase"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="genericRight" type="java.lang.Object"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[<code>Group</code> of counters, comprising of counters from a particular
+ counter {@link Enum} class.
+
+ <p><code>Group</code>handles localization of the class name and the
+ counter names.</p>]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.Counters.Group -->
+ <!-- start class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+ <class name="FileAlreadyExistsException" extends="java.io.IOException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="FileAlreadyExistsException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="FileAlreadyExistsException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <doc>
+ <![CDATA[Used when target file already exists for any operation and
+ is not configured to be overwritten.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+ <!-- start class org.apache.hadoop.mapred.FileInputFormat -->
+ <class name="FileInputFormat" extends="java.lang.Object"
+ abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapred.InputFormat"/>
+ <constructor name="FileInputFormat"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setMinSplitSize"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="minSplitSize" type="long"/>
+ </method>
+ <method name="isSplitable" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+ <param name="filename" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Is the given filename splitable? Usually, true, but if the file is
+ stream compressed, it will not be.
+
+ <code>FileInputFormat</code> implementations can override this and return
+ <code>false</code> to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+
+ @param fs the file system that the file is on
+ @param filename the file name to check
+ @return is this file splitable?]]>
+ </doc>
+ </method>
+ <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setInputPathFilter"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="filter" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Set a PathFilter to be applied to the input paths for the map-reduce job.
+
+ @param filter the PathFilter class use for filtering the input paths.]]>
+ </doc>
+ </method>
+ <method name="getInputPathFilter" return="org.apache.hadoop.fs.PathFilter"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Get a PathFilter instance of the filter set for the input paths.
+
+ @return the PathFilter instance set for the job, NULL if none has been set.]]>
+ </doc>
+ </method>
+ <method name="addInputPathRecursively"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="result" type="java.util.List"/>
+ <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <param name="inputFilter" type="org.apache.hadoop.fs.PathFilter"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Add files in the input path recursively into the results.
+ @param result
+ The List to store all files.
+ @param fs
+ The FileSystem.
+ @param path
+ The input path.
+ @param inputFilter
+ The input filter that can be used to filter files/dirs.
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression.
+
+ @param job the job to list input paths for
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+ </doc>
+ </method>
+ <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="file" type="org.apache.hadoop.fs.Path"/>
+ <param name="start" type="long"/>
+ <param name="length" type="long"/>
+ <param name="hosts" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+ </doc>
+ </method>
+ <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="file" type="org.apache.hadoop.fs.Path"/>
+ <param name="start" type="long"/>
+ <param name="length" type="long"/>
+ <param name="hosts" type="java.lang.String[]"/>
+ <param name="inMemoryHosts" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+ </doc>
+ </method>
+ <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="numSplits" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Splits files returned by {@link #listStatus(JobConf)} when
+ they're too big.]]>
+ </doc>
+ </method>
+ <method name="computeSplitSize" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="goalSize" type="long"/>
+ <param name="minSize" type="long"/>
+ <param name="blockSize" type="long"/>
+ </method>
+ <method name="getBlockIndex" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+ <param name="offset" type="long"/>
+ </method>
+ <method name="setInputPaths"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="commaSeparatedPaths" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Sets the given comma separated paths as the list of inputs
+ for the map-reduce job.
+
+ @param conf Configuration of the job
+ @param commaSeparatedPaths Comma separated paths to be set as
+ the list of inputs for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="addInputPaths"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="commaSeparatedPaths" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add the given comma separated paths to the list of inputs for
+ the map-reduce job.
+
+ @param conf The configuration of the job
+ @param commaSeparatedPaths Comma separated paths to be added to
+ the list of inputs for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="setInputPaths"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="inputPaths" type="org.apache.hadoop.fs.Path[]"/>
+ <doc>
+ <![CDATA[Set the array of {@link Path}s as the list of inputs
+ for the map-reduce job.
+
+ @param conf Configuration of the job.
+ @param inputPaths the {@link Path}s of the input directories/files
+ for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="addInputPath"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Add a {@link Path} to the list of inputs for the map-reduce job.
+
+ @param conf The configuration of the job
+ @param path {@link Path} to be added to the list of inputs for
+ the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="getInputPaths" return="org.apache.hadoop.fs.Path[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Get the list of input {@link Path}s for the map-reduce job.
+
+ @param conf The configuration of the job
+ @return the list of input {@link Path}s for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="getSplitHosts" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+ <param name="offset" type="long"/>
+ <param name="splitSize" type="long"/>
+ <param name="clusterMap" type="org.apache.hadoop.net.NetworkTopology"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[This function identifies and returns the hosts that contribute
+ most for a given split. For calculating the contribution, rack
+ locality is treated on par with host locality, so hosts from racks
+ that contribute the most are preferred over hosts on racks that
+ contribute less
+ @param blkLocations The list of block locations
+ @param offset
+ @param splitSize
+ @return an array of hosts that contribute most to this split
+ @throws IOException]]>
+ </doc>
+ </method>
+ <field name="LOG" type="org.apache.commons.logging.Log"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="NUM_INPUT_FILES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="INPUT_DIR_RECURSIVE" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[A base class for file-based {@link InputFormat}.
+
+ <p><code>FileInputFormat</code> is the base class for all file-based
+ <code>InputFormat</code>s. This provides a generic implementation of
+ {@link #getSplits(JobConf, int)}.
+ Subclasses of <code>FileInputFormat</code> can also override the
+ {@link #isSplitable(FileSystem, Path)} method to ensure input-files are
+ not split-up and are processed as a whole by {@link Mapper}s.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileInputFormat -->
+ <!-- start class org.apache.hadoop.mapred.FileInputFormat.Counter -->
+ <class name="FileInputFormat.Counter" extends="java.lang.Enum"
+ abstract="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.mapred.FileInputFormat.Counter[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.mapred.FileInputFormat.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileInputFormat.Counter -->
+ <!-- start class org.apache.hadoop.mapred.FileOutputCommitter -->
+ <class name="FileOutputCommitter" extends="org.apache.hadoop.mapred.OutputCommitter"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="FileOutputCommitter"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getTaskAttemptPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <param name="outputPath" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setupJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="commitJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="cleanupJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="abortJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <param name="runState" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setupTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="commitTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="abortTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="needsTaskCommit" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="isRecoverySupported" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="isRecoverySupported" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="recoverTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <field name="LOG" type="org.apache.commons.logging.Log"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="TEMP_DIR_NAME" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Temporary directory name]]>
+ </doc>
+ </field>
+ <field name="SUCCEEDED_FILE_NAME" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[An {@link OutputCommitter} that commits files specified
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileOutputCommitter -->
+ <!-- start class org.apache.hadoop.mapred.FileOutputFormat -->
+ <class name="FileOutputFormat" extends="java.lang.Object"
+ abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+ <constructor name="FileOutputFormat"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setCompressOutput"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="compress" type="boolean"/>
+ <doc>
+ <![CDATA[Set whether the output of the job is compressed.
+ @param conf the {@link JobConf} to modify
+ @param compress should the output of the job be compressed?]]>
+ </doc>
+ </method>
+ <method name="getCompressOutput" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Is the job output compressed?
+ @param conf the {@link JobConf} to look in
+ @return <code>true</code> if the job output should be compressed,
+ <code>false</code> otherwise]]>
+ </doc>
+ </method>
+ <method name="setOutputCompressorClass"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="codecClass" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Set the {@link CompressionCodec} to be used to compress job outputs.
+ @param conf the {@link JobConf} to modify
+ @param codecClass the {@link CompressionCodec} to be used to
+ compress the job outputs]]>
+ </doc>
+ </method>
+ <method name="getOutputCompressorClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="defaultValue" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the {@link CompressionCodec} for compressing the job outputs.
+ @param conf the {@link JobConf} to look in
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} to be used to compress the
+ job outputs
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+ </doc>
+ </method>
+ <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="name" type="java.lang.String"/>
+ <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="checkOutputSpecs"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <exception name="FileAlreadyExistsException" type="org.apache.hadoop.mapred.FileAlreadyExistsException"/>
+ <exception name="InvalidJobConfException" type="org.apache.hadoop.mapred.InvalidJobConfException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setOutputPath"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Set the {@link Path} of the output directory for the map-reduce job.
+
+ @param conf The configuration of the job.
+ @param outputDir the {@link Path} of the output directory for
+ the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="setWorkOutputPath"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Set the {@link Path} of the task's temporary output directory
+ for the map-reduce job.
+
+ <p><i>Note</i>: Task output path is set by the framework.
+ </p>
+ @param conf The configuration of the job.
+ @param outputDir the {@link Path} of the output directory
+ for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Get the {@link Path} to the output directory for the map-reduce job.
+
+ @return the {@link Path} to the output directory for the map-reduce job.
+ @see FileOutputFormat#getWorkOutputPath(JobConf)]]>
+ </doc>
+ </method>
+ <method name="getWorkOutputPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Get the {@link Path} to the task's temporary output directory
+ for the map-reduce job
+
+ <b id="SideEffectFiles">Tasks' Side-Effect Files</b>
+
+ <p><i>Note:</i> The following is valid only if the {@link OutputCommitter}
+ is {@link FileOutputCommitter}. If <code>OutputCommitter</code> is not
+ a <code>FileOutputCommitter</code>, the task's temporary output
+ directory is same as {@link #getOutputPath(JobConf)} i.e.
+ <tt>${mapreduce.output.fileoutputformat.outputdir}$</tt></p>
+
+ <p>Some applications need to create/write-to side-files, which differ from
+ the actual job-outputs.
+
+ <p>In such cases there could be issues with 2 instances of the same TIP
+ (running simultaneously e.g. speculative tasks) trying to open/write-to the
+ same file (path) on HDFS. Hence the application-writer will have to pick
+ unique names per task-attempt (e.g. using the attemptid, say
+ <tt>attempt_200709221812_0001_m_000000_0</tt>), not just per TIP.</p>
+
+ <p>To get around this the Map-Reduce framework helps the application-writer
+ out by maintaining a special
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt>
+ sub-directory for each task-attempt on HDFS where the output of the
+ task-attempt goes. On successful completion of the task-attempt the files
+ in the <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> (only)
+ are <i>promoted</i> to <tt>${mapreduce.output.fileoutputformat.outputdir}</tt>. Of course, the
+ framework discards the sub-directory of unsuccessful task-attempts. This
+ is completely transparent to the application.</p>
+
+ <p>The application-writer can take advantage of this by creating any
+ side-files required in <tt>${mapreduce.task.output.dir}</tt> during execution
+ of his reduce-task i.e. via {@link #getWorkOutputPath(JobConf)}, and the
+ framework will move them out similarly - thus she doesn't have to pick
+ unique paths per task-attempt.</p>
+
+ <p><i>Note</i>: the value of <tt>${mapreduce.task.output.dir}</tt> during
+ execution of a particular task-attempt is actually
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}</tt>, and this value is
+ set by the map-reduce framework. So, just create any side-files in the
+ path returned by {@link #getWorkOutputPath(JobConf)} from map/reduce
+ task to take advantage of this feature.</p>
+
+ <p>The entire discussion holds true for maps of jobs with
+ reducer=NONE (i.e. 0 reduces) since output of the map, in that case,
+ goes directly to HDFS.</p>
+
+ @return the {@link Path} to the task's temporary output directory
+ for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="getTaskOutputPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="name" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Helper function to create the task's temporary output directory and
+ return the path to the task's output file.
+
+ @param conf job-configuration
+ @param name temporary task-output filename
+ @return path to the task's temporary output file
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getUniqueName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Helper function to generate a name that is unique for the task.
+
+ <p>The generated name can be used to create custom files from within the
+ different tasks for the job, the names for different tasks will not collide
+ with each other.</p>
+
+ <p>The given name is postfixed with the task type, 'm' for maps, 'r' for
+ reduces and the task partition number. For example, give a name 'test'
+ running on the first map o the job the generated name will be
+ 'test-m-00000'.</p>
+
+ @param conf the configuration for the job.
+ @param name the name to make unique.
+ @return a unique name accross all tasks of the job.]]>
+ </doc>
+ </method>
+ <method name="getPathForCustomFile" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Helper function to generate a {@link Path} for a file that is unique for
+ the task within the job output directory.
+
+ <p>The path can be used to create custom files from within the map and
+ reduce tasks. The path name will be unique for each task. The path parent
+ will be the job output directory.</p>ls
+
+ <p>This method uses the {@link #getUniqueName} method to make the file name
+ unique for the task.</p>
+
+ @param conf the configuration for the job.
+ @param name the name for the file.
+ @return a unique path accross all tasks of the job.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[A base class for {@link OutputFormat}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileOutputFormat -->
+ <!-- start class org.apache.hadoop.mapred.FileOutputFormat.Counter -->
+ <class name="FileOutputFormat.Counter" extends="java.lang.Enum"
+ abstract="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.mapred.FileOutputFormat.Counter[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.mapred.FileOutputFormat.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileOutputFormat.Counter -->
+ <!-- start class org.apache.hadoop.mapred.FileSplit -->
+ <class name="FileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapred.InputSplitWithLocationInfo"/>
+ <constructor name="FileSplit"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, org.apache.hadoop.mapred.JobConf"
+ static="false" final="false" visibility="public"
+ deprec
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java
deleted file mode 100644
index 34d78a5..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import org.apache.hadoop.fs.TestFileContext;
-
-/**
- * Implementation of TestFileContext for S3a
- */
-public class TestS3AFileContext extends TestFileContext{
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java
deleted file mode 100644
index b0c4d84..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-
-/**
- * Extends FileContextCreateMkdirBaseTest for a S3a FileContext
- */
-public class TestS3AFileContextCreateMkdir
- extends FileContextCreateMkdirBaseTest {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- super.setUp();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java
deleted file mode 100644
index 4d200d1..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * S3A implementation of FileContextMainOperationsBaseTest
- */
-public class TestS3AFileContextMainOperations
- extends FileContextMainOperationsBaseTest {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- super.setUp();
- }
-
- @Override
- protected boolean listCorruptedBlocksSupported() {
- return false;
- }
-
- @Test
- @Ignore
- public void testCreateFlagAppendExistingFile() throws IOException {
- //append not supported, so test removed
- }
-
- @Test
- @Ignore
- public void testCreateFlagCreateAppendExistingFile() throws IOException {
- //append not supported, so test removed
- }
-
- @Test
- @Ignore
- public void testSetVerifyChecksum() throws IOException {
- //checksums ignored, so test removed
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java
deleted file mode 100644
index a9f4848..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.net.URI;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FCStatisticsBaseTest;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-
-/**
- * S3a implementation of FCStatisticsBaseTest
- */
-public class TestS3AFileContextStatistics extends FCStatisticsBaseTest {
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true);
- }
-
- @After
- public void tearDown() throws Exception {
- fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true);
- }
-
- @Override
- protected void verifyReadBytes(FileSystem.Statistics stats) {
- // one blockSize for read, one for pread
- Assert.assertEquals(2 * blockSize, stats.getBytesRead());
- }
-
- @Override
- protected void verifyWrittenBytes(FileSystem.Statistics stats) {
- //No extra bytes are written
- Assert.assertEquals(blockSize, stats.getBytesWritten());
- }
-
- @Override
- protected URI getFsUri() {
- return fc.getHomeDirectory().toUri();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java
deleted file mode 100644
index 3da7b19..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextURIBase;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * S3a implementation of FileContextURIBase
- */
-public class TestS3AFileContextURI extends FileContextURIBase {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc1 = S3ATestUtils.createTestFileContext(conf);
- fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
- super.setUp();
- }
-
- @Test
- @Ignore
- public void testFileStatus() throws IOException {
- //test disabled (the statistics tested with this method are not relevant for an S3FS)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java
deleted file mode 100644
index 666f4c2..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextUtilBase;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-
-/**
- * S3A implementation of FileContextUtilBase
- */
-public class TestS3AFileContextUtil extends FileContextUtilBase {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- super.setUp();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java
new file mode 100644
index 0000000..a375664
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Tests file deletion with multi-delete disabled.
+ */
+public class ITestS3ADeleteFilesOneByOne extends ITestS3ADeleteManyFiles {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration configuration = super.createConfiguration();
+ configuration.setBoolean(Constants.ENABLE_MULTI_DELETE, false);
+ return configuration;
+ }
+
+ @Override
+ @Test
+ public void testOpenCreate() throws IOException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
new file mode 100644
index 0000000..208c491
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * Test some scalable operations related to file renaming and deletion.
+ */
+public class ITestS3ADeleteManyFiles extends S3AScaleTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3ADeleteManyFiles.class);
+
+ /**
+ * CAUTION: If this test starts failing, please make sure that the
+ * {@link org.apache.hadoop.fs.s3a.Constants#MAX_THREADS} configuration is not
+ * set too low. Alternatively, consider reducing the
+ * <code>scale.test.operation.count</code> parameter in
+ * <code>getOperationCount()</code>.
+ *
+ * @see #getOperationCount()
+ */
+ @Test
+ public void testBulkRenameAndDelete() throws Throwable {
+ final Path scaleTestDir = getTestPath();
+ final Path srcDir = new Path(scaleTestDir, "src");
+ final Path finalDir = new Path(scaleTestDir, "final");
+ final long count = getOperationCount();
+ ContractTestUtils.rm(fs, scaleTestDir, true, false);
+
+ fs.mkdirs(srcDir);
+ fs.mkdirs(finalDir);
+
+ int testBufferSize = fs.getConf()
+ .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
+ ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
+ // use Executor to speed up file creation
+ ExecutorService exec = Executors.newFixedThreadPool(16);
+ final ExecutorCompletionService<Boolean> completionService =
+ new ExecutorCompletionService<>(exec);
+ try {
+ final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');
+
+ for (int i = 0; i < count; ++i) {
+ final String fileName = "foo-" + i;
+ completionService.submit(new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws IOException {
+ ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
+ false, data);
+ return fs.exists(new Path(srcDir, fileName));
+ }
+ });
+ }
+ for (int i = 0; i < count; ++i) {
+ final Future<Boolean> future = completionService.take();
+ try {
+ if (!future.get()) {
+ LOG.warn("cannot create file");
+ }
+ } catch (ExecutionException e) {
+ LOG.warn("Error while uploading file", e.getCause());
+ throw e;
+ }
+ }
+ } finally {
+ exec.shutdown();
+ }
+
+ int nSrcFiles = fs.listStatus(srcDir).length;
+ fs.rename(srcDir, finalDir);
+ assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
+ new Path(srcDir, "foo-" + 0));
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
+ new Path(srcDir, "foo-" + count / 2));
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
+ new Path(srcDir, "foo-" + (count - 1)));
+ ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
+ new Path(finalDir, "foo-" + 0));
+ ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
+ new Path(finalDir, "foo-" + count/2));
+ ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
+ new Path(finalDir, "foo-" + (count-1)));
+
+ ContractTestUtils.assertDeleted(fs, finalDir, true, false);
+ }
+
+ @Test
+ public void testOpenCreate() throws IOException {
+ Path dir = new Path("/tests3a");
+ ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
+ ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
+ ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);
+
+
+ /*
+ Enable to test the multipart upload
+ try {
+ ContractTestUtils.createAndVerifyFile(fs, dir,
+ (long)6 * 1024 * 1024 * 1024);
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ */
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
new file mode 100644
index 0000000..b5f4eb3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Statistic;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.s3a.Statistic.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+
+/**
+ * Test the performance of listing files/directories.
+ */
+public class ITestS3ADirectoryPerformance extends S3AScaleTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ITestS3ADirectoryPerformance.class);
+
+ @Test
+ public void testListOperations() throws Throwable {
+ describe("Test recursive list operations");
+ final Path scaleTestDir = getTestPath();
+ final Path listDir = new Path(scaleTestDir, "lists");
+
+ // scale factor.
+ int scale = getConf().getInt(KEY_DIRECTORY_COUNT, DEFAULT_DIRECTORY_COUNT);
+ int width = scale;
+ int depth = scale;
+ int files = scale;
+ MetricDiff metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
+ MetricDiff listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
+ MetricDiff listContinueRequests =
+ new MetricDiff(fs, OBJECT_CONTINUE_LIST_REQUESTS);
+ MetricDiff listStatusCalls = new MetricDiff(fs, INVOCATION_LIST_FILES);
+ MetricDiff getFileStatusCalls =
+ new MetricDiff(fs, INVOCATION_GET_FILE_STATUS);
+ NanoTimer createTimer = new NanoTimer();
+ TreeScanResults created =
+ createSubdirs(fs, listDir, depth, width, files, 0);
+ // add some empty directories
+ int emptyDepth = 1 * scale;
+ int emptyWidth = 3 * scale;
+
+ created.add(createSubdirs(fs, listDir, emptyDepth, emptyWidth, 0,
+ 0, "empty", "f-", ""));
+ createTimer.end("Time to create %s", created);
+ LOG.info("Time per operation: {}",
+ toHuman(createTimer.nanosPerOperation(created.totalCount())));
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+
+ describe("Listing files via treewalk");
+ try {
+ // Scan the directory via an explicit tree walk.
+ // This is the baseline for any listing speedups.
+ NanoTimer treeWalkTimer = new NanoTimer();
+ TreeScanResults treewalkResults = treeWalk(fs, listDir);
+ treeWalkTimer.end("List status via treewalk of %s", created);
+
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+ assertEquals("Files found in listFiles(recursive=true) " +
+ " created=" + created + " listed=" + treewalkResults,
+ created.getFileCount(), treewalkResults.getFileCount());
+
+ describe("Listing files via listFiles(recursive=true)");
+ // listFiles() does the recursion internally
+ NanoTimer listFilesRecursiveTimer = new NanoTimer();
+
+ TreeScanResults listFilesResults = new TreeScanResults(
+ fs.listFiles(listDir, true));
+
+ listFilesRecursiveTimer.end("listFiles(recursive=true) of %s", created);
+ assertEquals("Files found in listFiles(recursive=true) " +
+ " created=" + created + " listed=" + listFilesResults,
+ created.getFileCount(), listFilesResults.getFileCount());
+
+ // only two list operations should have taken place
+ print(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+ assertEquals(listRequests.toString(), 2, listRequests.diff());
+ reset(metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+
+
+ } finally {
+ describe("deletion");
+ // deletion at the end of the run
+ NanoTimer deleteTimer = new NanoTimer();
+ fs.delete(listDir, true);
+ deleteTimer.end("Deleting directory tree");
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+ }
+ }
+
+ @Test
+ public void testTimeToStatEmptyDirectory() throws Throwable {
+ describe("Time to stat an empty directory");
+ Path path = new Path(getTestPath(), "empty");
+ fs.mkdirs(path);
+ timeToStatPath(path);
+ }
+
+ @Test
+ public void testTimeToStatNonEmptyDirectory() throws Throwable {
+ describe("Time to stat a non-empty directory");
+ Path path = new Path(getTestPath(), "dir");
+ fs.mkdirs(path);
+ touch(fs, new Path(path, "file"));
+ timeToStatPath(path);
+ }
+
+ @Test
+ public void testTimeToStatFile() throws Throwable {
+ describe("Time to stat a simple file");
+ Path path = new Path(getTestPath(), "file");
+ touch(fs, path);
+ timeToStatPath(path);
+ }
+
+ @Test
+ public void testTimeToStatRoot() throws Throwable {
+ describe("Time to stat the root path");
+ timeToStatPath(new Path("/"));
+ }
+
+ private void timeToStatPath(Path path) throws IOException {
+ describe("Timing getFileStatus(\"%s\")", path);
+ MetricDiff metadataRequests =
+ new MetricDiff(fs, Statistic.OBJECT_METADATA_REQUESTS);
+ MetricDiff listRequests =
+ new MetricDiff(fs, Statistic.OBJECT_LIST_REQUESTS);
+ long attempts = getOperationCount();
+ NanoTimer timer = new NanoTimer();
+ for (long l = 0; l < attempts; l++) {
+ fs.getFileStatus(path);
+ }
+ timer.end("Time to execute %d getFileStatusCalls", attempts);
+ LOG.info("Time per call: {}", toHuman(timer.nanosPerOperation(attempts)));
+ LOG.info("metadata: {}", metadataRequests);
+ LOG.info("metadata per operation {}", metadataRequests.diff() / attempts);
+ LOG.info("listObjects: {}", listRequests);
+ LOG.info("listObjects: per operation {}", listRequests.diff() / attempts);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
new file mode 100644
index 0000000..e2163c5
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
@@ -0,0 +1,534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AInputPolicy;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AInstrumentation;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.util.LineReader;
+import org.junit.After;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * Look at the performance of S3a operations.
+ */
+public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ITestS3AInputStreamPerformance.class);
+
+ private S3AFileSystem s3aFS;
+ private Path testData;
+ private S3AFileStatus testDataStatus;
+ private FSDataInputStream in;
+ private S3AInstrumentation.InputStreamStatistics streamStatistics;
+ public static final int BLOCK_SIZE = 32 * 1024;
+ public static final int BIG_BLOCK_SIZE = 256 * 1024;
+
+ /** Tests only run if the there is a named test file that can be read. */
+ private boolean testDataAvailable = true;
+ private String assumptionMessage = "test file";
+
+ /**
+ * Open the FS and the test data. The input stream is always set up here.
+ * @throws IOException IO Problems.
+ */
+ @Before
+ public void openFS() throws IOException {
+ Configuration conf = getConf();
+ conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024);
+ conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024);
+ String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
+ if (testFile.isEmpty()) {
+ assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;
+ testDataAvailable = false;
+ } else {
+ S3ATestUtils.useCSVDataEndpoint(conf);
+ testData = new Path(testFile);
+ Path path = this.testData;
+ bindS3aFS(path);
+ try {
+ testDataStatus = s3aFS.getFileStatus(this.testData);
+ } catch (IOException e) {
+ LOG.warn("Failed to read file {} specified in {}",
+ testFile, KEY_CSVTEST_FILE, e);
+ throw e;
+ }
+ }
+ }
+
+ private void bindS3aFS(Path path) throws IOException {
+ s3aFS = (S3AFileSystem) FileSystem.newInstance(path.toUri(), getConf());
+ }
+
+ /**
+ * Cleanup: close the stream, close the FS.
+ */
+ @After
+ public void cleanup() {
+ describe("cleanup");
+ IOUtils.closeStream(in);
+ IOUtils.closeStream(s3aFS);
+ }
+
+ /**
+ * Declare that the test requires the CSV test dataset.
+ */
+ private void requireCSVTestData() {
+ Assume.assumeTrue(assumptionMessage, testDataAvailable);
+ }
+
+ /**
+ * Open the test file with the read buffer specified in the setting.
+ * {@link #KEY_READ_BUFFER_SIZE}; use the {@code Normal} policy
+ * @return the stream, wrapping an S3a one
+ * @throws IOException IO problems
+ */
+ FSDataInputStream openTestFile() throws IOException {
+ return openTestFile(S3AInputPolicy.Normal, 0);
+ }
+
+ /**
+ * Open the test file with the read buffer specified in the setting
+ * {@link #KEY_READ_BUFFER_SIZE}.
+ * This includes the {@link #requireCSVTestData()} assumption; so
+ * if called before any FS op, will automatically skip the test
+ * if the CSV file is absent.
+ *
+ * @param inputPolicy input policy to use
+ * @param readahead readahead/buffer size
+ * @return the stream, wrapping an S3a one
+ * @throws IOException IO problems
+ */
+ FSDataInputStream openTestFile(S3AInputPolicy inputPolicy, long readahead)
+ throws IOException {
+ requireCSVTestData();
+ return openDataFile(s3aFS, this.testData, inputPolicy, readahead);
+ }
+
+ /**
+ * Open a test file with the read buffer specified in the setting
+ * {@link #KEY_READ_BUFFER_SIZE}.
+ *
+ * @param path path to open
+ * @param inputPolicy input policy to use
+ * @param readahead readahead/buffer size
+ * @return the stream, wrapping an S3a one
+ * @throws IOException IO problems
+ */
+ private FSDataInputStream openDataFile(S3AFileSystem fs,
+ Path path,
+ S3AInputPolicy inputPolicy,
+ long readahead) throws IOException {
+ int bufferSize = getConf().getInt(KEY_READ_BUFFER_SIZE,
+ DEFAULT_READ_BUFFER_SIZE);
+ S3AInputPolicy policy = fs.getInputPolicy();
+ fs.setInputPolicy(inputPolicy);
+ try {
+ FSDataInputStream stream = fs.open(path, bufferSize);
+ if (readahead >= 0) {
+ stream.setReadahead(readahead);
+ }
+ streamStatistics = getInputStreamStatistics(stream);
+ return stream;
+ } finally {
+ fs.setInputPolicy(policy);
+ }
+ }
+
+ /**
+ * Assert that the stream was only ever opened once.
+ */
+ protected void assertStreamOpenedExactlyOnce() {
+ assertOpenOperationCount(1);
+ }
+
+ /**
+ * Make an assertion count about the number of open operations.
+ * @param expected the expected number
+ */
+ private void assertOpenOperationCount(long expected) {
+ assertEquals("open operations in\n" + in,
+ expected, streamStatistics.openOperations);
+ }
+
+ /**
+ * Log how long an IOP took, by dividing the total time by the
+ * count of operations, printing in a human-readable form.
+ * @param operation operation being measured
+ * @param timer timing data
+ * @param count IOP count.
+ */
+ protected void logTimePerIOP(String operation,
+ NanoTimer timer,
+ long count) {
+ LOG.info("Time per {}: {} nS",
+ operation, toHuman(timer.duration() / count));
+ }
+
+ @Test
+ public void testTimeToOpenAndReadWholeFileBlocks() throws Throwable {
+ requireCSVTestData();
+ int blockSize = _1MB;
+ describe("Open the test file %s and read it in blocks of size %d",
+ testData, blockSize);
+ long len = testDataStatus.getLen();
+ in = openTestFile();
+ byte[] block = new byte[blockSize];
+ NanoTimer timer2 = new NanoTimer();
+ long count = 0;
+ // implicitly rounding down here
+ long blockCount = len / blockSize;
+ for (long i = 0; i < blockCount; i++) {
+ int offset = 0;
+ int remaining = blockSize;
+ NanoTimer blockTimer = new NanoTimer();
+ int reads = 0;
+ while (remaining > 0) {
+ int bytesRead = in.read(block, offset, remaining);
+ reads++;
+ if (bytesRead == 1) {
+ break;
+ }
+ remaining -= bytesRead;
+ offset += bytesRead;
+ count += bytesRead;
+ }
+ blockTimer.end("Reading block %d in %d reads", i, reads);
+ }
+ timer2.end("Time to read %d bytes in %d blocks", len, blockCount);
+ bandwidth(timer2, count);
+ logStreamStatistics();
+ }
+
+ @Test
+ public void testLazySeekEnabled() throws Throwable {
+ describe("Verify that seeks do not trigger any IO");
+ in = openTestFile();
+ long len = testDataStatus.getLen();
+ NanoTimer timer = new NanoTimer();
+ long blockCount = len / BLOCK_SIZE;
+ for (long i = 0; i < blockCount; i++) {
+ in.seek(in.getPos() + BLOCK_SIZE - 1);
+ }
+ in.seek(0);
+ blockCount++;
+ timer.end("Time to execute %d seeks", blockCount);
+ logTimePerIOP("seek()", timer, blockCount);
+ logStreamStatistics();
+ assertOpenOperationCount(0);
+ assertEquals("bytes read", 0, streamStatistics.bytesRead);
+ }
+
+ @Test
+ public void testReadaheadOutOfRange() throws Throwable {
+ try {
+ in = openTestFile();
+ in.setReadahead(-1L);
+ fail("Stream should have rejected the request "+ in);
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testReadWithNormalPolicy() throws Throwable {
+ describe("Read big blocks with a big readahead");
+ executeSeekReadSequence(BIG_BLOCK_SIZE, BIG_BLOCK_SIZE * 2,
+ S3AInputPolicy.Normal);
+ assertStreamOpenedExactlyOnce();
+ }
+
+ @Test
+ public void testDecompressionSequential128K() throws Throwable {
+ describe("Decompress with a 128K readahead");
+ executeDecompression(128 * 1024, S3AInputPolicy.Sequential);
+ assertStreamOpenedExactlyOnce();
+ }
+
+ /**
+ * Execute a decompression + line read with the given input policy.
+ * @param readahead byte readahead
+ * @param inputPolicy read policy
+ * @throws IOException IO Problems
+ */
+ private void executeDecompression(long readahead,
+ S3AInputPolicy inputPolicy) throws IOException {
+ CompressionCodecFactory factory
+ = new CompressionCodecFactory(getConf());
+ CompressionCodec codec = factory.getCodec(testData);
+ long bytesRead = 0;
+ int lines = 0;
+
+ FSDataInputStream objectIn = openTestFile(inputPolicy, readahead);
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ try (LineReader lineReader = new LineReader(
+ codec.createInputStream(objectIn), getConf())) {
+ Text line = new Text();
+ int read;
+ while ((read = lineReader.readLine(line)) > 0) {
+ bytesRead += read;
+ lines++;
+ }
+ } catch (EOFException eof) {
+ // done
+ }
+ timer.end("Time to read %d lines [%d bytes expanded, %d raw]" +
+ " with readahead = %d",
+ lines,
+ bytesRead,
+ testDataStatus.getLen(),
+ readahead);
+ logTimePerIOP("line read", timer, lines);
+ logStreamStatistics();
+ }
+
+ private void logStreamStatistics() {
+ LOG.info(String.format("Stream Statistics%n{}"), streamStatistics);
+ }
+
+ /**
+ * Execute a seek+read sequence.
+ * @param blockSize block size for seeks
+ * @param readahead what the readahead value of the stream should be
+ * @throws IOException IO problems
+ */
+ protected void executeSeekReadSequence(long blockSize,
+ long readahead,
+ S3AInputPolicy policy) throws IOException {
+ in = openTestFile(policy, readahead);
+ long len = testDataStatus.getLen();
+ NanoTimer timer = new NanoTimer();
+ long blockCount = len / blockSize;
+ LOG.info("Reading {} blocks, readahead = {}",
+ blockCount, readahead);
+ for (long i = 0; i < blockCount; i++) {
+ in.seek(in.getPos() + blockSize - 1);
+ // this is the read
+ assertTrue(in.read() >= 0);
+ }
+ timer.end("Time to execute %d seeks of distance %d with readahead = %d",
+ blockCount,
+ blockSize,
+ readahead);
+ logTimePerIOP("seek(pos + " + blockCount+"); read()", timer, blockCount);
+ LOG.info("Effective bandwidth {} MB/S",
+ timer.bandwidthDescription(streamStatistics.bytesRead -
+ streamStatistics.bytesSkippedOnSeek));
+ logStreamStatistics();
+ }
+
+ public static final int _4K = 4 * 1024;
+ public static final int _8K = 8 * 1024;
+ public static final int _16K = 16 * 1024;
+ public static final int _32K = 32 * 1024;
+ public static final int _64K = 64 * 1024;
+ public static final int _128K = 128 * 1024;
+ public static final int _256K = 256 * 1024;
+ public static final int _1MB = 1024 * 1024;
+ public static final int _2MB = 2 * _1MB;
+ public static final int _10MB = _1MB * 10;
+ public static final int _5MB = _1MB * 5;
+
+ private static final int[][] RANDOM_IO_SEQUENCE = {
+ {_2MB, _128K},
+ {_128K, _128K},
+ {_5MB, _64K},
+ {_1MB, _1MB},
+ };
+
+ @Test
+ public void testRandomIORandomPolicy() throws Throwable {
+ executeRandomIO(S3AInputPolicy.Random, (long) RANDOM_IO_SEQUENCE.length);
+ assertEquals("streams aborted in " + streamStatistics,
+ 0, streamStatistics.aborted);
+ }
+
+ @Test
+ public void testRandomIONormalPolicy() throws Throwable {
+ long expectedOpenCount = RANDOM_IO_SEQUENCE.length;
+ executeRandomIO(S3AInputPolicy.Normal, expectedOpenCount);
+ assertEquals("streams aborted in " + streamStatistics,
+ 4, streamStatistics.aborted);
+ }
+
+ /**
+ * Execute the random IO {@code readFully(pos, bytes[])} sequence defined by
+ * {@link #RANDOM_IO_SEQUENCE}. The stream is closed afterwards; that's used
+ * in the timing too
+ * @param policy read policy
+ * @param expectedOpenCount expected number of stream openings
+ * @throws IOException IO problems
+ * @return the timer
+ */
+ private ContractTestUtils.NanoTimer executeRandomIO(S3AInputPolicy policy,
+ long expectedOpenCount)
+ throws IOException {
+ describe("Random IO with policy \"%s\"", policy);
+ byte[] buffer = new byte[_1MB];
+ long totalBytesRead = 0;
+
+ in = openTestFile(policy, 0);
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ for (int[] action : RANDOM_IO_SEQUENCE) {
+ int position = action[0];
+ int range = action[1];
+ in.readFully(position, buffer, 0, range);
+ totalBytesRead += range;
+ }
+ int reads = RANDOM_IO_SEQUENCE.length;
+ timer.end("Time to execute %d reads of total size %d bytes",
+ reads,
+ totalBytesRead);
+ in.close();
+ assertOpenOperationCount(expectedOpenCount);
+ logTimePerIOP("byte read", timer, totalBytesRead);
+ LOG.info("Effective bandwidth {} MB/S",
+ timer.bandwidthDescription(streamStatistics.bytesRead -
+ streamStatistics.bytesSkippedOnSeek));
+ logStreamStatistics();
+ return timer;
+ }
+
+ S3AInputStream getS3aStream() {
+ return (S3AInputStream) in.getWrappedStream();
+ }
+
+ @Test
+ public void testRandomReadOverBuffer() throws Throwable {
+ describe("read over a buffer, making sure that the requests" +
+ " spans readahead ranges");
+ int datasetLen = _32K;
+ Path dataFile = new Path(getTestPath(), "testReadOverBuffer.bin");
+ byte[] sourceData = dataset(datasetLen, 0, 64);
+ // relies on the field 'fs' referring to the R/W FS
+ writeDataset(fs, dataFile, sourceData, datasetLen, _16K, true);
+ byte[] buffer = new byte[datasetLen];
+ int readahead = _8K;
+ int halfReadahead = _4K;
+ in = openDataFile(fs, dataFile, S3AInputPolicy.Random, readahead);
+
+ LOG.info("Starting initial reads");
+ S3AInputStream s3aStream = getS3aStream();
+ assertEquals(readahead, s3aStream.getReadahead());
+ byte[] oneByte = new byte[1];
+ assertEquals(1, in.read(0, oneByte, 0, 1));
+ // make some assertions about the current state
+ assertEquals("remaining in\n" + in,
+ readahead - 1, s3aStream.remainingInCurrentRequest());
+ assertEquals("range start in\n" + in,
+ 0, s3aStream.getContentRangeStart());
+ assertEquals("range finish in\n" + in,
+ readahead, s3aStream.getContentRangeFinish());
+
+ assertStreamOpenedExactlyOnce();
+
+ describe("Starting sequence of positioned read calls over\n%s", in);
+ NanoTimer readTimer = new NanoTimer();
+ int currentPos = halfReadahead;
+ int offset = currentPos;
+ int bytesRead = 0;
+ int readOps = 0;
+
+ // make multiple read() calls
+ while (bytesRead < halfReadahead) {
+ int length = buffer.length - offset;
+ int read = in.read(currentPos, buffer, offset, length);
+ bytesRead += read;
+ offset += read;
+ readOps++;
+ assertEquals("open operations on request #" + readOps
+ + " after reading " + bytesRead
+ + " current position in stream " + currentPos
+ + " in\n" + fs
+ + "\n " + in,
+ 1, streamStatistics.openOperations);
+ for (int i = currentPos; i < currentPos + read; i++) {
+ assertEquals("Wrong value from byte " + i,
+ sourceData[i], buffer[i]);
+ }
+ currentPos += read;
+ }
+ assertStreamOpenedExactlyOnce();
+ // assert at the end of the original block
+ assertEquals(readahead, currentPos);
+ readTimer.end("read %d in %d operations", bytesRead, readOps);
+ bandwidth(readTimer, bytesRead);
+ LOG.info("Time per byte(): {} nS",
+ toHuman(readTimer.nanosPerOperation(bytesRead)));
+ LOG.info("Time per read(): {} nS",
+ toHuman(readTimer.nanosPerOperation(readOps)));
+
+ describe("read last byte");
+ // read one more
+ int read = in.read(currentPos, buffer, bytesRead, 1);
+ assertTrue("-1 from last read", read >= 0);
+ assertOpenOperationCount(2);
+ assertEquals("Wrong value from read ", sourceData[currentPos],
+ (int) buffer[currentPos]);
+ currentPos++;
+
+
+ // now scan all the way to the end of the file, using single byte read()
+ // calls
+ describe("read() to EOF over \n%s", in);
+ long readCount = 0;
+ NanoTimer timer = new NanoTimer();
+ LOG.info("seeking");
+ in.seek(currentPos);
+ LOG.info("reading");
+ while(currentPos < datasetLen) {
+ int r = in.read();
+ assertTrue("Negative read() at position " + currentPos + " in\n" + in,
+ r >= 0);
+ buffer[currentPos] = (byte)r;
+ assertEquals("Wrong value from read from\n" + in,
+ sourceData[currentPos], r);
+ currentPos++;
+ readCount++;
+ }
+ timer.end("read %d bytes", readCount);
+ bandwidth(timer, readCount);
+ LOG.info("Time per read(): {} nS",
+ toHuman(timer.nanosPerOperation(readCount)));
+
+ assertEquals("last read in " + in, -1, in.read());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADeleteManyFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADeleteManyFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADeleteManyFiles.java
deleted file mode 100644
index 5e07dcb..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADeleteManyFiles.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.scale;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- * Test some scalable operations related to file renaming and deletion.
- */
-public class TestS3ADeleteManyFiles extends S3AScaleTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3ADeleteManyFiles.class);
-
- /**
- * CAUTION: If this test starts failing, please make sure that the
- * {@link org.apache.hadoop.fs.s3a.Constants#MAX_THREADS} configuration is not
- * set too low. Alternatively, consider reducing the
- * <code>scale.test.operation.count</code> parameter in
- * <code>getOperationCount()</code>.
- *
- * @see #getOperationCount()
- */
- @Test
- public void testBulkRenameAndDelete() throws Throwable {
- final Path scaleTestDir = getTestPath();
- final Path srcDir = new Path(scaleTestDir, "src");
- final Path finalDir = new Path(scaleTestDir, "final");
- final long count = getOperationCount();
- ContractTestUtils.rm(fs, scaleTestDir, true, false);
-
- fs.mkdirs(srcDir);
- fs.mkdirs(finalDir);
-
- int testBufferSize = fs.getConf()
- .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
- ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
- // use Executor to speed up file creation
- ExecutorService exec = Executors.newFixedThreadPool(16);
- final ExecutorCompletionService<Boolean> completionService =
- new ExecutorCompletionService<>(exec);
- try {
- final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');
-
- for (int i = 0; i < count; ++i) {
- final String fileName = "foo-" + i;
- completionService.submit(new Callable<Boolean>() {
- @Override
- public Boolean call() throws IOException {
- ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
- false, data);
- return fs.exists(new Path(srcDir, fileName));
- }
- });
- }
- for (int i = 0; i < count; ++i) {
- final Future<Boolean> future = completionService.take();
- try {
- if (!future.get()) {
- LOG.warn("cannot create file");
- }
- } catch (ExecutionException e) {
- LOG.warn("Error while uploading file", e.getCause());
- throw e;
- }
- }
- } finally {
- exec.shutdown();
- }
-
- int nSrcFiles = fs.listStatus(srcDir).length;
- fs.rename(srcDir, finalDir);
- assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
- ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
- new Path(srcDir, "foo-" + 0));
- ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
- new Path(srcDir, "foo-" + count / 2));
- ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
- new Path(srcDir, "foo-" + (count - 1)));
- ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
- new Path(finalDir, "foo-" + 0));
- ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
- new Path(finalDir, "foo-" + count/2));
- ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
- new Path(finalDir, "foo-" + (count-1)));
-
- ContractTestUtils.assertDeleted(fs, finalDir, true, false);
- }
-
- @Test
- public void testOpenCreate() throws IOException {
- Path dir = new Path("/tests3a");
- ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
- ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
- ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);
-
-
- /*
- Enable to test the multipart upload
- try {
- ContractTestUtils.createAndVerifyFile(fs, dir,
- (long)6 * 1024 * 1024 * 1024);
- } catch (IOException e) {
- fail(e.getMessage());
- }
- */
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
deleted file mode 100644
index 35ea3ad..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.scale;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.Statistic;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.s3a.Statistic.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-
-/**
- * Test the performance of listing files/directories.
- */
-public class TestS3ADirectoryPerformance extends S3AScaleTestBase {
- private static final Logger LOG = LoggerFactory.getLogger(
- TestS3ADirectoryPerformance.class);
-
- @Test
- public void testListOperations() throws Throwable {
- describe("Test recursive list operations");
- final Path scaleTestDir = getTestPath();
- final Path listDir = new Path(scaleTestDir, "lists");
-
- // scale factor.
- int scale = getConf().getInt(KEY_DIRECTORY_COUNT, DEFAULT_DIRECTORY_COUNT);
- int width = scale;
- int depth = scale;
- int files = scale;
- MetricDiff metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
- MetricDiff listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
- MetricDiff listContinueRequests =
- new MetricDiff(fs, OBJECT_CONTINUE_LIST_REQUESTS);
- MetricDiff listStatusCalls = new MetricDiff(fs, INVOCATION_LIST_FILES);
- MetricDiff getFileStatusCalls =
- new MetricDiff(fs, INVOCATION_GET_FILE_STATUS);
- NanoTimer createTimer = new NanoTimer();
- TreeScanResults created =
- createSubdirs(fs, listDir, depth, width, files, 0);
- // add some empty directories
- int emptyDepth = 1 * scale;
- int emptyWidth = 3 * scale;
-
- created.add(createSubdirs(fs, listDir, emptyDepth, emptyWidth, 0,
- 0, "empty", "f-", ""));
- createTimer.end("Time to create %s", created);
- LOG.info("Time per operation: {}",
- toHuman(createTimer.nanosPerOperation(created.totalCount())));
- printThenReset(LOG,
- metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
-
- describe("Listing files via treewalk");
- try {
- // Scan the directory via an explicit tree walk.
- // This is the baseline for any listing speedups.
- NanoTimer treeWalkTimer = new NanoTimer();
- TreeScanResults treewalkResults = treeWalk(fs, listDir);
- treeWalkTimer.end("List status via treewalk of %s", created);
-
- printThenReset(LOG,
- metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
- assertEquals("Files found in listFiles(recursive=true) " +
- " created=" + created + " listed=" + treewalkResults,
- created.getFileCount(), treewalkResults.getFileCount());
-
- describe("Listing files via listFiles(recursive=true)");
- // listFiles() does the recursion internally
- NanoTimer listFilesRecursiveTimer = new NanoTimer();
-
- TreeScanResults listFilesResults = new TreeScanResults(
- fs.listFiles(listDir, true));
-
- listFilesRecursiveTimer.end("listFiles(recursive=true) of %s", created);
- assertEquals("Files found in listFiles(recursive=true) " +
- " created=" + created + " listed=" + listFilesResults,
- created.getFileCount(), listFilesResults.getFileCount());
-
- // only two list operations should have taken place
- print(LOG,
- metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
- assertEquals(listRequests.toString(), 2, listRequests.diff());
- reset(metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
-
-
- } finally {
- describe("deletion");
- // deletion at the end of the run
- NanoTimer deleteTimer = new NanoTimer();
- fs.delete(listDir, true);
- deleteTimer.end("Deleting directory tree");
- printThenReset(LOG,
- metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
- }
- }
-
- @Test
- public void testTimeToStatEmptyDirectory() throws Throwable {
- describe("Time to stat an empty directory");
- Path path = new Path(getTestPath(), "empty");
- fs.mkdirs(path);
- timeToStatPath(path);
- }
-
- @Test
- public void testTimeToStatNonEmptyDirectory() throws Throwable {
- describe("Time to stat a non-empty directory");
- Path path = new Path(getTestPath(), "dir");
- fs.mkdirs(path);
- touch(fs, new Path(path, "file"));
- timeToStatPath(path);
- }
-
- @Test
- public void testTimeToStatFile() throws Throwable {
- describe("Time to stat a simple file");
- Path path = new Path(getTestPath(), "file");
- touch(fs, path);
- timeToStatPath(path);
- }
-
- @Test
- public void testTimeToStatRoot() throws Throwable {
- describe("Time to stat the root path");
- timeToStatPath(new Path("/"));
- }
-
- private void timeToStatPath(Path path) throws IOException {
- describe("Timing getFileStatus(\"%s\")", path);
- MetricDiff metadataRequests =
- new MetricDiff(fs, Statistic.OBJECT_METADATA_REQUESTS);
- MetricDiff listRequests =
- new MetricDiff(fs, Statistic.OBJECT_LIST_REQUESTS);
- long attempts = getOperationCount();
- NanoTimer timer = new NanoTimer();
- for (long l = 0; l < attempts; l++) {
- fs.getFileStatus(path);
- }
- timer.end("Time to execute %d getFileStatusCalls", attempts);
- LOG.info("Time per call: {}", toHuman(timer.nanosPerOperation(attempts)));
- LOG.info("metadata: {}", metadataRequests);
- LOG.info("metadata per operation {}", metadataRequests.diff() / attempts);
- LOG.info("listObjects: {}", listRequests);
- LOG.info("listObjects: per operation {}", listRequests.diff() / attempts);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
deleted file mode 100644
index d6d9d66..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
+++ /dev/null
@@ -1,534 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.scale;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3AInputPolicy;
-import org.apache.hadoop.fs.s3a.S3AInputStream;
-import org.apache.hadoop.fs.s3a.S3AInstrumentation;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.util.LineReader;
-import org.junit.After;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-
-/**
- * Look at the performance of S3a operations.
- */
-public class TestS3AInputStreamPerformance extends S3AScaleTestBase {
- private static final Logger LOG = LoggerFactory.getLogger(
- TestS3AInputStreamPerformance.class);
-
- private S3AFileSystem s3aFS;
- private Path testData;
- private S3AFileStatus testDataStatus;
- private FSDataInputStream in;
- private S3AInstrumentation.InputStreamStatistics streamStatistics;
- public static final int BLOCK_SIZE = 32 * 1024;
- public static final int BIG_BLOCK_SIZE = 256 * 1024;
-
- /** Tests only run if the there is a named test file that can be read. */
- private boolean testDataAvailable = true;
- private String assumptionMessage = "test file";
-
- /**
- * Open the FS and the test data. The input stream is always set up here.
- * @throws IOException IO Problems.
- */
- @Before
- public void openFS() throws IOException {
- Configuration conf = getConf();
- conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024);
- conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024);
- String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
- if (testFile.isEmpty()) {
- assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;
- testDataAvailable = false;
- } else {
- S3ATestUtils.useCSVDataEndpoint(conf);
- testData = new Path(testFile);
- Path path = this.testData;
- bindS3aFS(path);
- try {
- testDataStatus = s3aFS.getFileStatus(this.testData);
- } catch (IOException e) {
- LOG.warn("Failed to read file {} specified in {}",
- testFile, KEY_CSVTEST_FILE, e);
- throw e;
- }
- }
- }
-
- private void bindS3aFS(Path path) throws IOException {
- s3aFS = (S3AFileSystem) FileSystem.newInstance(path.toUri(), getConf());
- }
-
- /**
- * Cleanup: close the stream, close the FS.
- */
- @After
- public void cleanup() {
- describe("cleanup");
- IOUtils.closeStream(in);
- IOUtils.closeStream(s3aFS);
- }
-
- /**
- * Declare that the test requires the CSV test dataset.
- */
- private void requireCSVTestData() {
- Assume.assumeTrue(assumptionMessage, testDataAvailable);
- }
-
- /**
- * Open the test file with the read buffer specified in the setting.
- * {@link #KEY_READ_BUFFER_SIZE}; use the {@code Normal} policy
- * @return the stream, wrapping an S3a one
- * @throws IOException IO problems
- */
- FSDataInputStream openTestFile() throws IOException {
- return openTestFile(S3AInputPolicy.Normal, 0);
- }
-
- /**
- * Open the test file with the read buffer specified in the setting
- * {@link #KEY_READ_BUFFER_SIZE}.
- * This includes the {@link #requireCSVTestData()} assumption; so
- * if called before any FS op, will automatically skip the test
- * if the CSV file is absent.
- *
- * @param inputPolicy input policy to use
- * @param readahead readahead/buffer size
- * @return the stream, wrapping an S3a one
- * @throws IOException IO problems
- */
- FSDataInputStream openTestFile(S3AInputPolicy inputPolicy, long readahead)
- throws IOException {
- requireCSVTestData();
- return openDataFile(s3aFS, this.testData, inputPolicy, readahead);
- }
-
- /**
- * Open a test file with the read buffer specified in the setting
- * {@link #KEY_READ_BUFFER_SIZE}.
- *
- * @param path path to open
- * @param inputPolicy input policy to use
- * @param readahead readahead/buffer size
- * @return the stream, wrapping an S3a one
- * @throws IOException IO problems
- */
- private FSDataInputStream openDataFile(S3AFileSystem fs,
- Path path,
- S3AInputPolicy inputPolicy,
- long readahead) throws IOException {
- int bufferSize = getConf().getInt(KEY_READ_BUFFER_SIZE,
- DEFAULT_READ_BUFFER_SIZE);
- S3AInputPolicy policy = fs.getInputPolicy();
- fs.setInputPolicy(inputPolicy);
- try {
- FSDataInputStream stream = fs.open(path, bufferSize);
- if (readahead >= 0) {
- stream.setReadahead(readahead);
- }
- streamStatistics = getInputStreamStatistics(stream);
- return stream;
- } finally {
- fs.setInputPolicy(policy);
- }
- }
-
- /**
- * Assert that the stream was only ever opened once.
- */
- protected void assertStreamOpenedExactlyOnce() {
- assertOpenOperationCount(1);
- }
-
- /**
- * Make an assertion count about the number of open operations.
- * @param expected the expected number
- */
- private void assertOpenOperationCount(long expected) {
- assertEquals("open operations in\n" + in,
- expected, streamStatistics.openOperations);
- }
-
- /**
- * Log how long an IOP took, by dividing the total time by the
- * count of operations, printing in a human-readable form.
- * @param operation operation being measured
- * @param timer timing data
- * @param count IOP count.
- */
- protected void logTimePerIOP(String operation,
- NanoTimer timer,
- long count) {
- LOG.info("Time per {}: {} nS",
- operation, toHuman(timer.duration() / count));
- }
-
- @Test
- public void testTimeToOpenAndReadWholeFileBlocks() throws Throwable {
- requireCSVTestData();
- int blockSize = _1MB;
- describe("Open the test file %s and read it in blocks of size %d",
- testData, blockSize);
- long len = testDataStatus.getLen();
- in = openTestFile();
- byte[] block = new byte[blockSize];
- NanoTimer timer2 = new NanoTimer();
- long count = 0;
- // implicitly rounding down here
- long blockCount = len / blockSize;
- for (long i = 0; i < blockCount; i++) {
- int offset = 0;
- int remaining = blockSize;
- NanoTimer blockTimer = new NanoTimer();
- int reads = 0;
- while (remaining > 0) {
- int bytesRead = in.read(block, offset, remaining);
- reads ++;
- if (bytesRead == 1) {
- break;
- }
- remaining -= bytesRead;
- offset += bytesRead;
- count += bytesRead;
- }
- blockTimer.end("Reading block %d in %d reads", i, reads);
- }
- timer2.end("Time to read %d bytes in %d blocks", len, blockCount );
- bandwidth(timer2, count);
- logStreamStatistics();
- }
-
- @Test
- public void testLazySeekEnabled() throws Throwable {
- describe("Verify that seeks do not trigger any IO");
- in = openTestFile();
- long len = testDataStatus.getLen();
- NanoTimer timer = new NanoTimer();
- long blockCount = len / BLOCK_SIZE;
- for (long i = 0; i < blockCount; i++) {
- in.seek(in.getPos() + BLOCK_SIZE - 1);
- }
- in.seek(0);
- blockCount++;
- timer.end("Time to execute %d seeks", blockCount);
- logTimePerIOP("seek()", timer, blockCount);
- logStreamStatistics();
- assertOpenOperationCount(0);
- assertEquals("bytes read", 0, streamStatistics.bytesRead);
- }
-
- @Test
- public void testReadaheadOutOfRange() throws Throwable {
- try {
- in = openTestFile();
- in.setReadahead(-1L);
- fail("Stream should have rejected the request "+ in);
- } catch (IllegalArgumentException e) {
- // expected
- }
- }
-
- @Test
- public void testReadWithNormalPolicy() throws Throwable {
- describe("Read big blocks with a big readahead");
- executeSeekReadSequence(BIG_BLOCK_SIZE, BIG_BLOCK_SIZE * 2,
- S3AInputPolicy.Normal);
- assertStreamOpenedExactlyOnce();
- }
-
- @Test
- public void testDecompressionSequential128K() throws Throwable {
- describe("Decompress with a 128K readahead");
- executeDecompression(128 * 1024, S3AInputPolicy.Sequential);
- assertStreamOpenedExactlyOnce();
- }
-
- /**
- * Execute a decompression + line read with the given input policy.
- * @param readahead byte readahead
- * @param inputPolicy read policy
- * @throws IOException IO Problems
- */
- private void executeDecompression(long readahead,
- S3AInputPolicy inputPolicy) throws IOException {
- CompressionCodecFactory factory
- = new CompressionCodecFactory(getConf());
- CompressionCodec codec = factory.getCodec(testData);
- long bytesRead = 0;
- int lines = 0;
-
- FSDataInputStream objectIn = openTestFile(inputPolicy, readahead);
- ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
- try (LineReader lineReader = new LineReader(
- codec.createInputStream(objectIn), getConf())) {
- Text line = new Text();
- int read;
- while ((read = lineReader.readLine(line)) > 0) {
- bytesRead += read;
- lines++;
- }
- } catch (EOFException eof) {
- // done
- }
- timer.end("Time to read %d lines [%d bytes expanded, %d raw]" +
- " with readahead = %d",
- lines,
- bytesRead,
- testDataStatus.getLen(),
- readahead);
- logTimePerIOP("line read", timer, lines);
- logStreamStatistics();
- }
-
- private void logStreamStatistics() {
- LOG.info(String.format("Stream Statistics%n{}"), streamStatistics);
- }
-
- /**
- * Execute a seek+read sequence.
- * @param blockSize block size for seeks
- * @param readahead what the readahead value of the stream should be
- * @throws IOException IO problems
- */
- protected void executeSeekReadSequence(long blockSize,
- long readahead,
- S3AInputPolicy policy) throws IOException {
- in = openTestFile(policy, readahead);
- long len = testDataStatus.getLen();
- NanoTimer timer = new NanoTimer();
- long blockCount = len / blockSize;
- LOG.info("Reading {} blocks, readahead = {}",
- blockCount, readahead);
- for (long i = 0; i < blockCount; i++) {
- in.seek(in.getPos() + blockSize - 1);
- // this is the read
- assertTrue(in.read() >= 0);
- }
- timer.end("Time to execute %d seeks of distance %d with readahead = %d",
- blockCount,
- blockSize,
- readahead);
- logTimePerIOP("seek(pos + " + blockCount+"); read()", timer, blockCount);
- LOG.info("Effective bandwidth {} MB/S",
- timer.bandwidthDescription(streamStatistics.bytesRead -
- streamStatistics.bytesSkippedOnSeek));
- logStreamStatistics();
- }
-
- public static final int _4K = 4 * 1024;
- public static final int _8K = 8 * 1024;
- public static final int _16K = 16 * 1024;
- public static final int _32K = 32 * 1024;
- public static final int _64K = 64 * 1024;
- public static final int _128K = 128 * 1024;
- public static final int _256K = 256 * 1024;
- public static final int _1MB = 1024 * 1024;
- public static final int _2MB = 2 * _1MB;
- public static final int _10MB = _1MB * 10;
- public static final int _5MB = _1MB * 5;
-
- private static final int[][] RANDOM_IO_SEQUENCE = {
- {_2MB, _128K},
- {_128K, _128K},
- {_5MB, _64K},
- {_1MB, _1MB},
- };
-
- @Test
- public void testRandomIORandomPolicy() throws Throwable {
- executeRandomIO(S3AInputPolicy.Random, (long) RANDOM_IO_SEQUENCE.length);
- assertEquals("streams aborted in " + streamStatistics,
- 0, streamStatistics.aborted);
- }
-
- @Test
- public void testRandomIONormalPolicy() throws Throwable {
- long expectedOpenCount = RANDOM_IO_SEQUENCE.length;
- executeRandomIO(S3AInputPolicy.Normal, expectedOpenCount);
- assertEquals("streams aborted in " + streamStatistics,
- 4, streamStatistics.aborted);
- }
-
- /**
- * Execute the random IO {@code readFully(pos, bytes[])} sequence defined by
- * {@link #RANDOM_IO_SEQUENCE}. The stream is closed afterwards; that's used
- * in the timing too
- * @param policy read policy
- * @param expectedOpenCount expected number of stream openings
- * @throws IOException IO problems
- * @return the timer
- */
- private ContractTestUtils.NanoTimer executeRandomIO(S3AInputPolicy policy,
- long expectedOpenCount)
- throws IOException {
- describe("Random IO with policy \"%s\"", policy);
- byte[] buffer = new byte[_1MB];
- long totalBytesRead = 0;
-
- in = openTestFile(policy, 0);
- ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
- for (int[] action : RANDOM_IO_SEQUENCE) {
- int position = action[0];
- int range = action[1];
- in.readFully(position, buffer, 0, range);
- totalBytesRead += range;
- }
- int reads = RANDOM_IO_SEQUENCE.length;
- timer.end("Time to execute %d reads of total size %d bytes",
- reads,
- totalBytesRead);
- in.close();
- assertOpenOperationCount(expectedOpenCount);
- logTimePerIOP("byte read", timer, totalBytesRead);
- LOG.info("Effective bandwidth {} MB/S",
- timer.bandwidthDescription(streamStatistics.bytesRead -
- streamStatistics.bytesSkippedOnSeek));
- logStreamStatistics();
- return timer;
- }
-
- S3AInputStream getS3aStream() {
- return (S3AInputStream) in.getWrappedStream();
- }
-
- @Test
- public void testRandomReadOverBuffer() throws Throwable {
- describe("read over a buffer, making sure that the requests" +
- " spans readahead ranges");
- int datasetLen = _32K;
- Path dataFile = new Path(getTestPath(), "testReadOverBuffer.bin");
- byte[] sourceData = dataset(datasetLen, 0, 64);
- // relies on the field 'fs' referring to the R/W FS
- writeDataset(fs, dataFile, sourceData, datasetLen, _16K, true);
- byte[] buffer = new byte[datasetLen];
- int readahead = _8K;
- int halfReadahead = _4K;
- in = openDataFile(fs, dataFile, S3AInputPolicy.Random, readahead);
-
- LOG.info("Starting initial reads");
- S3AInputStream s3aStream = getS3aStream();
- assertEquals(readahead, s3aStream.getReadahead());
- byte[] oneByte = new byte[1];
- assertEquals(1, in.read(0, oneByte, 0, 1));
- // make some assertions about the current state
- assertEquals("remaining in\n" + in,
- readahead - 1, s3aStream.remainingInCurrentRequest());
- assertEquals("range start in\n" + in,
- 0, s3aStream.getContentRangeStart());
- assertEquals("range finish in\n" + in,
- readahead, s3aStream.getContentRangeFinish());
-
- assertStreamOpenedExactlyOnce();
-
- describe("Starting sequence of positioned read calls over\n%s", in);
- NanoTimer readTimer = new NanoTimer();
- int currentPos = halfReadahead;
- int offset = currentPos;
- int bytesRead = 0;
- int readOps = 0;
-
- // make multiple read() calls
- while (bytesRead < halfReadahead) {
- int length = buffer.length - offset;
- int read = in.read(currentPos, buffer, offset, length);
- bytesRead += read;
- offset += read;
- readOps++;
- assertEquals("open operations on request #" + readOps
- + " after reading " + bytesRead
- + " current position in stream " + currentPos
- + " in\n" + fs
- + "\n " + in,
- 1, streamStatistics.openOperations);
- for (int i = currentPos; i < currentPos + read; i++) {
- assertEquals("Wrong value from byte " + i,
- sourceData[i], buffer[i]);
- }
- currentPos += read;
- }
- assertStreamOpenedExactlyOnce();
- // assert at the end of the original block
- assertEquals(readahead, currentPos);
- readTimer.end("read %d in %d operations", bytesRead, readOps);
- bandwidth(readTimer, bytesRead);
- LOG.info("Time per byte(): {} nS",
- toHuman(readTimer.nanosPerOperation(bytesRead)));
- LOG.info("Time per read(): {} nS",
- toHuman(readTimer.nanosPerOperation(readOps)));
-
- describe("read last byte");
- // read one more
- int read = in.read(currentPos, buffer, bytesRead, 1);
- assertTrue("-1 from last read", read >= 0);
- assertOpenOperationCount(2);
- assertEquals("Wrong value from read ", sourceData[currentPos],
- (int) buffer[currentPos]);
- currentPos++;
-
-
- // now scan all the way to the end of the file, using single byte read()
- // calls
- describe("read() to EOF over \n%s", in);
- long readCount = 0;
- NanoTimer timer = new NanoTimer();
- LOG.info("seeking");
- in.seek(currentPos);
- LOG.info("reading");
- while(currentPos < datasetLen) {
- int r = in.read();
- assertTrue("Negative read() at position " + currentPos + " in\n" + in,
- r >= 0);
- buffer[currentPos] = (byte)r;
- assertEquals("Wrong value from read from\n" + in,
- sourceData[currentPos], r);
- currentPos++;
- readCount++;
- }
- timer.end("read %d bytes", readCount);
- bandwidth(timer, readCount);
- LOG.info("Time per read(): {} nS",
- toHuman(timer.nanosPerOperation(readCount)));
-
- assertEquals("last read in " + in, -1, in.read());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java
new file mode 100644
index 0000000..ca57da6
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FsStatus;
+import org.apache.hadoop.fs.Path;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.util.EnumSet;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * S3A tests through the {@link FileContext} API.
+ */
+public class ITestS3A {
+ private FileContext fc;
+
+ @Rule
+ public final Timeout testTimeout = new Timeout(90000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fc != null) {
+ fc.delete(getTestPath(), true);
+ }
+ }
+
+ protected Path getTestPath() {
+ return new Path("/tests3afc");
+ }
+
+ @Test
+ public void testS3AStatus() throws Exception {
+ FsStatus fsStatus = fc.getFsStatus(null);
+ assertNotNull(fsStatus);
+ assertTrue("Used capacity should be positive: " + fsStatus.getUsed(),
+ fsStatus.getUsed() >= 0);
+ assertTrue("Remaining capacity should be positive: " + fsStatus
+ .getRemaining(),
+ fsStatus.getRemaining() >= 0);
+ assertTrue("Capacity should be positive: " + fsStatus.getCapacity(),
+ fsStatus.getCapacity() >= 0);
+ }
+
+ @Test
+ public void testS3ACreateFileInSubDir() throws Exception {
+ Path dirPath = getTestPath();
+ fc.mkdir(dirPath, FileContext.DIR_DEFAULT_PERM, true);
+ Path filePath = new Path(dirPath, "file");
+ try (FSDataOutputStream file = fc.create(filePath, EnumSet.of(CreateFlag
+ .CREATE))) {
+ file.write(666);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
new file mode 100644
index 0000000..772d8c7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.fs.s3a.yarn;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.WordCount;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+
+import org.junit.After;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests that S3A is usable through a YARN application.
+ */
+public class ITestS3AMiniYarnCluster {
+
+ private final Configuration conf = new YarnConfiguration();
+ private S3AFileSystem fs;
+ private MiniYARNCluster yarnCluster;
+ private final String rootPath = "/tests/MiniClusterWordCount/";
+
+ @Before
+ public void beforeTest() throws IOException {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fs.mkdirs(new Path(rootPath + "input/"));
+
+ yarnCluster = new MiniYARNCluster("MiniClusterWordCount", // testName
+ 1, // number of node managers
+ 1, // number of local log dirs per node manager
+ 1); // number of hdfs dirs per node manager
+ yarnCluster.init(conf);
+ yarnCluster.start();
+ }
+
+ @After
+ public void afterTest() throws IOException {
+ fs.delete(new Path(rootPath), true);
+ yarnCluster.stop();
+ }
+
+ @Test
+ public void testWithMiniCluster() throws Exception {
+ Path input = new Path(rootPath + "input/in.txt");
+ input = input.makeQualified(fs.getUri(), fs.getWorkingDirectory());
+ Path output = new Path(rootPath + "output/");
+ output = output.makeQualified(fs.getUri(), fs.getWorkingDirectory());
+
+ writeStringToFile(input, "first line\nsecond line\nthird line");
+
+ Job job = Job.getInstance(conf, "word count");
+ job.setJarByClass(WordCount.class);
+ job.setMapperClass(WordCount.TokenizerMapper.class);
+ job.setCombinerClass(WordCount.IntSumReducer.class);
+ job.setReducerClass(WordCount.IntSumReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+ FileInputFormat.addInputPath(job, input);
+ FileOutputFormat.setOutputPath(job, output);
+
+ int exitCode = (job.waitForCompletion(true) ? 0 : 1);
+ assertEquals("Returned error code.", 0, exitCode);
+
+ assertTrue(fs.exists(new Path(output, "_SUCCESS")));
+ String outputAsStr = readStringFromFile(new Path(output, "part-r-00000"));
+ Map<String, Integer> resAsMap = getResultAsMap(outputAsStr);
+
+ assertEquals(4, resAsMap.size());
+ assertEquals(1, (int) resAsMap.get("first"));
+ assertEquals(1, (int) resAsMap.get("second"));
+ assertEquals(1, (int) resAsMap.get("third"));
+ assertEquals(3, (int) resAsMap.get("line"));
+ }
+
+ /**
+ * helper method.
+ */
+ private Map<String, Integer> getResultAsMap(String outputAsStr)
+ throws IOException {
+ Map<String, Integer> result = new HashMap<>();
+ for (String line : outputAsStr.split("\n")) {
+ String[] tokens = line.split("\t");
+ result.put(tokens[0], Integer.parseInt(tokens[1]));
+ }
+ return result;
+ }
+
+ /**
+ * helper method.
+ */
+ private void writeStringToFile(Path path, String string) throws IOException {
+ FileContext fc = S3ATestUtils.createTestFileContext(conf);
+ try (FSDataOutputStream file = fc.create(path,
+ EnumSet.of(CreateFlag.CREATE))) {
+ file.write(string.getBytes());
+ }
+ }
+
+ /**
+ * helper method.
+ */
+ private String readStringFromFile(Path path) {
+ try (FSDataInputStream in = fs.open(path)) {
+ long bytesLen = fs.getFileStatus(path).getLen();
+ byte[] buffer = new byte[(int) bytesLen];
+ IOUtils.readFully(in, buffer, 0, buffer.length);
+ return new String(buffer);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to read from [" + path + "]", e);
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: MAPREDUCE-6310. Add jdiff support
to MapReduce. (Li Lu/vinodkv via wangda)
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.6.0.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.6.0.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.6.0.xml
new file mode 100644
index 0000000..14f8307
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.6.0.xml
@@ -0,0 +1,30767 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon Jun 13 17:08:13 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="hadoop-mapreduce-client-core 2.6.0"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-annotations.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/jdiff.jar -verbose -classpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/classes:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-2.6.0.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-2.6.0.jar:/Users/vinodkv/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/
Users/vinodkv/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/vinodkv/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/vinodkv/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/vinodkv/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/vinodkv/.m2/repository/org/co
dehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/vinodkv/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/vinodkv/.m2/repository/com/google/inject/guice/3.0/guice-3.0.jar:/Users/vinodkv/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/Users/vinodkv/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/vinodkv/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/vinodkv/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/Users/vinodkv/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/vinodkv/
.m2/repository/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/Users/vinodkv/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/vinodkv/.m2/repository/org/htrace/htrace-core/3.0.4/htrace-core-3.0.4.jar:/Users/vinodkv/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/vinodkv/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/vinodkv/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/vinodkv/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-common/target/hadoop-common-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/vinodkv/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/vinodkv/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/vinodkv/.m2/repository/net/java/dev/je
ts3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/vinodkv/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/vinodkv/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/vinodkv/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/vinodkv/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-kerb
eros-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-framework/2.6.0/curator-framework-2.6.0.jar:/Users/vinodkv/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-client/2.6.0/curator-client-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-recipes/2.6.0/curator-recipes-2.6.0.jar:/Users/vinodkv/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/Users/vinodkv/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/vinodkv/.
m2/repository/org/slf4j/slf4j-log4j12/1.7.5/slf4j-log4j12-1.7.5.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.6.0.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home/lib/tools.jar:/Users/vinodkv/.m2/repository/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar:/Users/vinodkv/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/vinodkv/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/vinodkv/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/vinodkv/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/vinodkv/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/vinodkv/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/vinodkv/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar -sourcepath /Us
ers/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java -apidir /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/site/jdiff/xml -apiname hadoop-mapreduce-client-core 2.6.0 -->
+<package name="org.apache.hadoop.filecache">
+ <!-- start class org.apache.hadoop.filecache.DistributedCache -->
+ <class name="DistributedCache" extends="org.apache.hadoop.mapreduce.filecache.DistributedCache"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="DistributedCache"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="addLocalArchives"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="str" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a archive that has been localized to the conf. Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+ </doc>
+ </method>
+ <method name="addLocalFiles"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="str" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a file that has been localized to the conf.. Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+ </doc>
+ </method>
+ <method name="createAllSymlink"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="Internal to MapReduce framework. Use DistributedCacheManager
+ instead.">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="jobCacheDir" type="java.io.File"/>
+ <param name="workDir" type="java.io.File"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[This method create symlinks for all files in a given dir in another
+ directory. Currently symlinks cannot be disabled. This is a NO-OP.
+
+ @param conf the configuration
+ @param jobCacheDir the target directory for creating symlinks
+ @param workDir the directory in which the symlinks are created
+ @throws IOException
+ @deprecated Internal to MapReduce framework. Use DistributedCacheManager
+ instead.]]>
+ </doc>
+ </method>
+ <method name="getFileStatus" return="org.apache.hadoop.fs.FileStatus"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="cache" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns {@link FileStatus} of a given cache file on hdfs. Internal to
+ MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return <code>FileStatus</code> of a given cache file on hdfs
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getTimestamp" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="cache" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns mtime of a given cache file on hdfs. Internal to MapReduce.
+ @param conf configuration
+ @param cache cache file
+ @return mtime of a given cache file on hdfs
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="setArchiveTimestamps"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="timestamps" type="java.lang.String"/>
+ <doc>
+ <![CDATA[This is to check the timestamp of the archives to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of archives.
+ The order should be the same as the order in which the archives are added.]]>
+ </doc>
+ </method>
+ <method name="setFileTimestamps"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="timestamps" type="java.lang.String"/>
+ <doc>
+ <![CDATA[This is to check the timestamp of the files to be localized.
+ Used by internal MapReduce code.
+ @param conf Configuration which stores the timestamp's
+ @param timestamps comma separated list of timestamps of files.
+ The order should be the same as the order in which the files are added.]]>
+ </doc>
+ </method>
+ <method name="setLocalArchives"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="str" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the conf to contain the location for localized archives. Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local archives]]>
+ </doc>
+ </method>
+ <method name="setLocalFiles"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="str" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the conf to contain the location for localized files. Used
+ by internal DistributedCache code.
+ @param conf The conf to modify to contain the localized caches
+ @param str a comma separated list of local files]]>
+ </doc>
+ </method>
+ <field name="CACHE_FILES_SIZES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_FILES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES_SIZES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_ARCHIVES_SIZES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_ARCHIVES_SIZES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_SIZES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_ARCHIVES_TIMESTAMPS" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_ARCHIVES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES_TIMESTAMPS}]]>
+ </doc>
+ </field>
+ <field name="CACHE_FILES_TIMESTAMPS" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_FILES_TIMESTAMPS} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILE_TIMESTAMPS}]]>
+ </doc>
+ </field>
+ <field name="CACHE_ARCHIVES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_ARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_ARCHIVES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_FILES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_FILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_FILES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_LOCALARCHIVES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_LOCALARCHIVES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALARCHIVES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_LOCALFILES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_LOCALFILES} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_LOCALFILES}]]>
+ </doc>
+ </field>
+ <field name="CACHE_SYMLINK" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Warning: {@link #CACHE_SYMLINK} is not a *public* constant.
+ The variable is kept for M/R 1.x applications, M/R 2.x applications should
+ use {@link MRJobConfig#CACHE_SYMLINK}]]>
+ </doc>
+ </field>
+ <doc>
+ <![CDATA[Distribute application-specific large, read-only files efficiently.
+
+ <p><code>DistributedCache</code> is a facility provided by the Map-Reduce
+ framework to cache files (text, archives, jars etc.) needed by applications.
+ </p>
+
+ <p>Applications specify the files, via urls (hdfs:// or http://) to be cached
+ via the {@link org.apache.hadoop.mapred.JobConf}. The
+ <code>DistributedCache</code> assumes that the files specified via urls are
+ already present on the {@link FileSystem} at the path specified by the url
+ and are accessible by every machine in the cluster.</p>
+
+ <p>The framework will copy the necessary files on to the slave node before
+ any tasks for the job are executed on that node. Its efficiency stems from
+ the fact that the files are only copied once per job and the ability to
+ cache archives which are un-archived on the slaves.</p>
+
+ <p><code>DistributedCache</code> can be used to distribute simple, read-only
+ data/text files and/or more complex types such as archives, jars etc.
+ Archives (zip, tar and tgz/tar.gz files) are un-archived at the slave nodes.
+ Jars may be optionally added to the classpath of the tasks, a rudimentary
+ software distribution mechanism. Files have execution permissions.
+ In older version of Hadoop Map/Reduce users could optionally ask for symlinks
+ to be created in the working directory of the child task. In the current
+ version symlinks are always created. If the URL does not have a fragment
+ the name of the file or directory will be used. If multiple files or
+ directories map to the same link name, the last one added, will be used. All
+ others will not even be downloaded.</p>
+
+ <p><code>DistributedCache</code> tracks modification timestamps of the cache
+ files. Clearly the cache files should not be modified by the application
+ or externally while the job is executing.</p>
+
+ <p>Here is an illustrative example on how to use the
+ <code>DistributedCache</code>:</p>
+ <p><blockquote><pre>
+ // Setting up the cache for the application
+
+ 1. Copy the requisite files to the <code>FileSystem</code>:
+
+ $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat
+ $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip
+ $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
+ $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
+ $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
+ $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
+
+ 2. Setup the application's <code>JobConf</code>:
+
+ JobConf job = new JobConf();
+ DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"),
+ job);
+ DistributedCache.addCacheArchive(new URI("/myapp/map.zip", job);
+ DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
+ DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar", job);
+ DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz", job);
+ DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz", job);
+
+ 3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
+ or {@link org.apache.hadoop.mapred.Reducer}:
+
+ public static class MapClass extends MapReduceBase
+ implements Mapper<K, V, K, V> {
+
+ private Path[] localArchives;
+ private Path[] localFiles;
+
+ public void configure(JobConf job) {
+ // Get the cached archives/files
+ File f = new File("./map.zip/some/file/in/zip.txt");
+ }
+
+ public void map(K key, V value,
+ OutputCollector<K, V> output, Reporter reporter)
+ throws IOException {
+ // Use data from the cached archives/files here
+ // ...
+ // ...
+ output.collect(k, v);
+ }
+ }
+
+ </pre></blockquote></p>
+
+ It is also very common to use the DistributedCache by using
+ {@link org.apache.hadoop.util.GenericOptionsParser}.
+
+ This class includes methods that should be used by users
+ (specifically those mentioned in the example above, as well
+ as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}),
+ as well as methods intended for use by the MapReduce framework
+ (e.g., {@link org.apache.hadoop.mapred.JobClient}).
+
+ @see org.apache.hadoop.mapred.JobConf
+ @see org.apache.hadoop.mapred.JobClient
+ @see org.apache.hadoop.mapreduce.Job]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.filecache.DistributedCache -->
+</package>
+<package name="org.apache.hadoop.mapred">
+ <!-- start class org.apache.hadoop.mapred.ClusterStatus -->
+ <class name="ClusterStatus" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.io.Writable"/>
+ <method name="getTaskTrackers" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of task trackers in the cluster.
+
+ @return the number of task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getActiveTrackerNames" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the names of task trackers in the cluster.
+
+ @return the active task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getBlacklistedTrackerNames" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the names of task trackers in the cluster.
+
+ @return the blacklisted task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getGraylistedTrackerNames" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the names of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return an empty graylisted task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getGraylistedTrackers" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of graylisted task trackers in the cluster.
+
+ The gray list of trackers is no longer available on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return 0 graylisted task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getBlacklistedTrackers" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of blacklisted task trackers in the cluster.
+
+ @return the number of blacklisted task trackers in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getNumExcludedNodes" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of excluded hosts in the cluster.
+ @return the number of excluded hosts in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getTTExpiryInterval" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the tasktracker expiry interval for the cluster
+ @return the expiry interval in msec]]>
+ </doc>
+ </method>
+ <method name="getMapTasks" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of currently running map tasks in the cluster.
+
+ @return the number of currently running map tasks in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getReduceTasks" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the number of currently running reduce tasks in the cluster.
+
+ @return the number of currently running reduce tasks in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getMaxMapTasks" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the maximum capacity for running map tasks in the cluster.
+
+ @return the maximum capacity for running map tasks in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getMaxReduceTasks" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the maximum capacity for running reduce tasks in the cluster.
+
+ @return the maximum capacity for running reduce tasks in the cluster.]]>
+ </doc>
+ </method>
+ <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the JobTracker's status.
+
+ @return {@link JobTrackerStatus} of the JobTracker]]>
+ </doc>
+ </method>
+ <method name="getMaxMemory" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+ </doc>
+ </method>
+ <method name="getUsedMemory" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Returns UNINITIALIZED_MEMORY_VALUE (-1)]]>
+ </doc>
+ </method>
+ <method name="getBlackListedTrackersInfo" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets the list of blacklisted trackers along with reasons for blacklisting.
+
+ @return the collection of {@link BlackListInfo} objects.]]>
+ </doc>
+ </method>
+ <method name="getJobTrackerState" return="org.apache.hadoop.mapred.JobTracker.State"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the current state of the <code>JobTracker</code>,
+ as {@link JobTracker.State}
+
+ {@link JobTracker.State} should no longer be used on M/R 2.x. The function
+ is kept to be compatible with M/R 1.x applications.
+
+ @return the invalid state of the <code>JobTracker</code>.]]>
+ </doc>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <field name="UNINITIALIZED_MEMORY_VALUE" type="long"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[Status information on the current state of the Map-Reduce cluster.
+
+ <p><code>ClusterStatus</code> provides clients with information such as:
+ <ol>
+ <li>
+ Size of the cluster.
+ </li>
+ <li>
+ Name of the trackers.
+ </li>
+ <li>
+ Task capacity of the cluster.
+ </li>
+ <li>
+ The number of currently running map & reduce tasks.
+ </li>
+ <li>
+ State of the <code>JobTracker</code>.
+ </li>
+ <li>
+ Details regarding black listed trackers.
+ </li>
+ </ol></p>
+
+ <p>Clients can query for the latest <code>ClusterStatus</code>, via
+ {@link JobClient#getClusterStatus()}.</p>
+
+ @see JobClient]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ClusterStatus -->
+ <!-- start class org.apache.hadoop.mapred.ClusterStatus.BlackListInfo -->
+ <class name="ClusterStatus.BlackListInfo" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.io.Writable"/>
+ <method name="getTrackerName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets the blacklisted tasktracker's name.
+
+ @return tracker's name.]]>
+ </doc>
+ </method>
+ <method name="getReasonForBlackListing" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets the reason for which the tasktracker was blacklisted.
+
+ @return reason which tracker was blacklisted]]>
+ </doc>
+ </method>
+ <method name="getBlackListReport" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets a descriptive report about why the tasktracker was blacklisted.
+
+ @return report describing why the tasktracker was blacklisted.]]>
+ </doc>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Class which encapsulates information about a blacklisted tasktracker.
+
+ The information includes the tasktracker's name and reasons for
+ getting blacklisted. The toString method of the class will print
+ the information in a whitespace separated fashion to enable parsing.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ClusterStatus.BlackListInfo -->
+ <!-- start class org.apache.hadoop.mapred.Counters -->
+ <class name="Counters" extends="org.apache.hadoop.mapreduce.counters.AbstractCounters"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Counters"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="Counters" type="org.apache.hadoop.mapreduce.Counters"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getGroup" return="org.apache.hadoop.mapred.Counters.Group"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="groupName" type="java.lang.String"/>
+ </method>
+ <method name="getGroupNames" return="java.util.Collection"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="makeCompactString" return="java.lang.String"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="group" type="java.lang.String"/>
+ <param name="name" type="java.lang.String"/>
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="use {@link #findCounter(String, String)} instead">
+ <param name="group" type="java.lang.String"/>
+ <param name="id" type="int"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Find a counter by using strings
+ @param group the name of the group
+ @param id the id of the counter within the group (0 to N-1)
+ @param name the internal name of the counter
+ @return the counter for that name
+ @deprecated use {@link #findCounter(String, String)} instead]]>
+ </doc>
+ </method>
+ <method name="incrCounter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.Enum"/>
+ <param name="amount" type="long"/>
+ <doc>
+ <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param key identifies a counter
+ @param amount amount by which counter is to be incremented]]>
+ </doc>
+ </method>
+ <method name="incrCounter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="group" type="java.lang.String"/>
+ <param name="counter" type="java.lang.String"/>
+ <param name="amount" type="long"/>
+ <doc>
+ <![CDATA[Increments the specified counter by the specified amount, creating it if
+ it didn't already exist.
+ @param group the name of the group
+ @param counter the internal name of the counter
+ @param amount amount by which counter is to be incremented]]>
+ </doc>
+ </method>
+ <method name="getCounter" return="long"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.Enum"/>
+ <doc>
+ <![CDATA[Returns current value of the specified counter, or 0 if the counter
+ does not exist.
+ @param key the counter enum to lookup
+ @return the counter value or 0 if counter not found]]>
+ </doc>
+ </method>
+ <method name="incrAllCounters"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="other" type="org.apache.hadoop.mapred.Counters"/>
+ <doc>
+ <![CDATA[Increments multiple counters by their amounts in another Counters
+ instance.
+ @param other the other Counters instance]]>
+ </doc>
+ </method>
+ <method name="size" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="use {@link #countCounters()} instead">
+ <doc>
+ <![CDATA[@return the total number of counters
+ @deprecated use {@link #countCounters()} instead]]>
+ </doc>
+ </method>
+ <method name="sum" return="org.apache.hadoop.mapred.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="a" type="org.apache.hadoop.mapred.Counters"/>
+ <param name="b" type="org.apache.hadoop.mapred.Counters"/>
+ <doc>
+ <![CDATA[Convenience method for computing the sum of two sets of counters.
+ @param a the first counters
+ @param b the second counters
+ @return a new summed counters object]]>
+ </doc>
+ </method>
+ <method name="log"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="log" type="org.apache.commons.logging.Log"/>
+ <doc>
+ <![CDATA[Logs the current counter values.
+ @param log The log to use.]]>
+ </doc>
+ </method>
+ <method name="makeEscapedCompactString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Represent the counter in a textual format that can be converted back to
+ its object form
+ @return the string in the following format
+ {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*]]>
+ </doc>
+ </method>
+ <method name="fromEscapedCompactString" return="org.apache.hadoop.mapred.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="compactString" type="java.lang.String"/>
+ <exception name="ParseException" type="java.text.ParseException"/>
+ <doc>
+ <![CDATA[Convert a stringified (by {@link #makeEscapedCompactString()} counter
+ representation into a counter object.
+ @param compactString to parse
+ @return a new counters object
+ @throws ParseException]]>
+ </doc>
+ </method>
+ <field name="MAX_COUNTER_LIMIT" type="int"
+ transient="false" volatile="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="MAX_GROUP_LIMIT" type="int"
+ transient="false" volatile="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[A set of named counters.
+
+ <p><code>Counters</code> represent global counters, defined either by the
+ Map-Reduce framework or applications. Each <code>Counter</code> can be of
+ any {@link Enum} type.</p>
+
+ <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
+ counters from a particular <code>Enum</code> class.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.Counters -->
+ <!-- start class org.apache.hadoop.mapred.Counters.Counter -->
+ <class name="Counters.Counter" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.Counter"/>
+ <constructor name="Counters.Counter"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setDisplayName"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="displayName" type="java.lang.String"/>
+ </method>
+ <method name="getName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDisplayName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getValue" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setValue"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="value" type="long"/>
+ </method>
+ <method name="increment"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="incr" type="long"/>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="makeEscapedCompactString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Returns the compact stringified version of the counter in the format
+ [(actual-name)(display-name)(value)]
+ @return the stringified result]]>
+ </doc>
+ </method>
+ <method name="contentEquals" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="deprecated, no comment">
+ <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+ <doc>
+ <![CDATA[Checks for (content) equality of two (basic) counters
+ @param counter to compare
+ @return true if content equals
+ @deprecated]]>
+ </doc>
+ </method>
+ <method name="getCounter" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return the value of the counter]]>
+ </doc>
+ </method>
+ <method name="getUnderlyingCounter" return="org.apache.hadoop.mapreduce.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="genericRight" type="java.lang.Object"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[A counter record, comprising its name and value.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.Counters.Counter -->
+ <!-- start class org.apache.hadoop.mapred.Counters.CountersExceededException -->
+ <class name="Counters.CountersExceededException" extends="java.lang.RuntimeException"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Counters.CountersExceededException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="Counters.CountersExceededException" type="org.apache.hadoop.mapred.Counters.CountersExceededException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <doc>
+ <![CDATA[Counter exception thrown when the number of counters exceed the limit]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.Counters.CountersExceededException -->
+ <!-- start class org.apache.hadoop.mapred.Counters.Group -->
+ <class name="Counters.Group" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+ <constructor name="Counters.Group"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getCounter" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counterName" type="java.lang.String"/>
+ <doc>
+ <![CDATA[@param counterName the name of the counter
+ @return the value of the specified counter, or 0 if the counter does
+ not exist.]]>
+ </doc>
+ </method>
+ <method name="makeEscapedCompactString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return the compact stringified version of the group in the format
+ {(actual-name)(display-name)(value)[][][]} where [] are compact strings
+ for the counters within.]]>
+ </doc>
+ </method>
+ <method name="getCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="use {@link #findCounter(String)} instead">
+ <param name="id" type="int"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the counter for the given id and create it if it doesn't exist.
+ @param id the numeric id of the counter within the group
+ @param name the internal counter name
+ @return the counter
+ @deprecated use {@link #findCounter(String)} instead]]>
+ </doc>
+ </method>
+ <method name="getCounterForName" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the counter for the given name and create it if it doesn't exist.
+ @param name the internal counter name
+ @return the counter]]>
+ </doc>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="iterator" return="java.util.Iterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDisplayName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setDisplayName"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="displayName" type="java.lang.String"/>
+ </method>
+ <method name="addCounter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counter" type="org.apache.hadoop.mapred.Counters.Counter"/>
+ </method>
+ <method name="addCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="displayName" type="java.lang.String"/>
+ <param name="value" type="long"/>
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counterName" type="java.lang.String"/>
+ <param name="displayName" type="java.lang.String"/>
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counterName" type="java.lang.String"/>
+ <param name="create" type="boolean"/>
+ </method>
+ <method name="findCounter" return="org.apache.hadoop.mapred.Counters.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="counterName" type="java.lang.String"/>
+ </method>
+ <method name="size" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="incrAllCounters"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="rightGroup" type="org.apache.hadoop.mapreduce.counters.CounterGroupBase"/>
+ </method>
+ <method name="getUnderlyingGroup" return="org.apache.hadoop.mapreduce.counters.CounterGroupBase"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="genericRight" type="java.lang.Object"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[<code>Group</code> of counters, comprising of counters from a particular
+ counter {@link Enum} class.
+
+ <p><code>Group</code>handles localization of the class name and the
+ counter names.</p>]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.Counters.Group -->
+ <!-- start class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+ <class name="FileAlreadyExistsException" extends="java.io.IOException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="FileAlreadyExistsException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="FileAlreadyExistsException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <doc>
+ <![CDATA[Used when target file already exists for any operation and
+ is not configured to be overwritten.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileAlreadyExistsException -->
+ <!-- start class org.apache.hadoop.mapred.FileInputFormat -->
+ <class name="FileInputFormat" extends="java.lang.Object"
+ abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapred.InputFormat"/>
+ <constructor name="FileInputFormat"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setMinSplitSize"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="minSplitSize" type="long"/>
+ </method>
+ <method name="isSplitable" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+ <param name="filename" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Is the given filename splitable? Usually, true, but if the file is
+ stream compressed, it will not be.
+
+ <code>FileInputFormat</code> implementations can override this and return
+ <code>false</code> to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+
+ @param fs the file system that the file is on
+ @param filename the file name to check
+ @return is this file splitable?]]>
+ </doc>
+ </method>
+ <method name="getRecordReader" return="org.apache.hadoop.mapred.RecordReader"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="split" type="org.apache.hadoop.mapred.InputSplit"/>
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="reporter" type="org.apache.hadoop.mapred.Reporter"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setInputPathFilter"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="filter" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Set a PathFilter to be applied to the input paths for the map-reduce job.
+
+ @param filter the PathFilter class use for filtering the input paths.]]>
+ </doc>
+ </method>
+ <method name="getInputPathFilter" return="org.apache.hadoop.fs.PathFilter"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Get a PathFilter instance of the filter set for the input paths.
+
+ @return the PathFilter instance set for the job, NULL if none has been set.]]>
+ </doc>
+ </method>
+ <method name="addInputPathRecursively"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="result" type="java.util.List"/>
+ <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <param name="inputFilter" type="org.apache.hadoop.fs.PathFilter"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Add files in the input path recursively into the results.
+ @param result
+ The List to store all files.
+ @param fs
+ The FileSystem.
+ @param path
+ The input path.
+ @param inputFilter
+ The input filter that can be used to filter files/dirs.
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="listStatus" return="org.apache.hadoop.fs.FileStatus[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[List input directories.
+ Subclasses may override to, e.g., select only files matching a regular
+ expression.
+
+ @param job the job to list input paths for
+ @return array of FileStatus objects
+ @throws IOException if zero items.]]>
+ </doc>
+ </method>
+ <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="file" type="org.apache.hadoop.fs.Path"/>
+ <param name="start" type="long"/>
+ <param name="length" type="long"/>
+ <param name="hosts" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+ </doc>
+ </method>
+ <method name="makeSplit" return="org.apache.hadoop.mapred.FileSplit"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="file" type="org.apache.hadoop.fs.Path"/>
+ <param name="start" type="long"/>
+ <param name="length" type="long"/>
+ <param name="hosts" type="java.lang.String[]"/>
+ <param name="inMemoryHosts" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[A factory that makes the split for this class. It can be overridden
+ by sub-classes to make sub-types]]>
+ </doc>
+ </method>
+ <method name="getSplits" return="org.apache.hadoop.mapred.InputSplit[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="numSplits" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Splits files returned by {@link #listStatus(JobConf)} when
+ they're too big.]]>
+ </doc>
+ </method>
+ <method name="computeSplitSize" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="goalSize" type="long"/>
+ <param name="minSize" type="long"/>
+ <param name="blockSize" type="long"/>
+ </method>
+ <method name="getBlockIndex" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+ <param name="offset" type="long"/>
+ </method>
+ <method name="setInputPaths"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="commaSeparatedPaths" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Sets the given comma separated paths as the list of inputs
+ for the map-reduce job.
+
+ @param conf Configuration of the job
+ @param commaSeparatedPaths Comma separated paths to be set as
+ the list of inputs for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="addInputPaths"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="commaSeparatedPaths" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add the given comma separated paths to the list of inputs for
+ the map-reduce job.
+
+ @param conf The configuration of the job
+ @param commaSeparatedPaths Comma separated paths to be added to
+ the list of inputs for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="setInputPaths"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="inputPaths" type="org.apache.hadoop.fs.Path[]"/>
+ <doc>
+ <![CDATA[Set the array of {@link Path}s as the list of inputs
+ for the map-reduce job.
+
+ @param conf Configuration of the job.
+ @param inputPaths the {@link Path}s of the input directories/files
+ for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="addInputPath"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Add a {@link Path} to the list of inputs for the map-reduce job.
+
+ @param conf The configuration of the job
+ @param path {@link Path} to be added to the list of inputs for
+ the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="getInputPaths" return="org.apache.hadoop.fs.Path[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Get the list of input {@link Path}s for the map-reduce job.
+
+ @param conf The configuration of the job
+ @return the list of input {@link Path}s for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="getSplitHosts" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="blkLocations" type="org.apache.hadoop.fs.BlockLocation[]"/>
+ <param name="offset" type="long"/>
+ <param name="splitSize" type="long"/>
+ <param name="clusterMap" type="org.apache.hadoop.net.NetworkTopology"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[This function identifies and returns the hosts that contribute
+ most for a given split. For calculating the contribution, rack
+ locality is treated on par with host locality, so hosts from racks
+ that contribute the most are preferred over hosts on racks that
+ contribute less
+ @param blkLocations The list of block locations
+ @param offset
+ @param splitSize
+ @return an array of hosts that contribute most to this split
+ @throws IOException]]>
+ </doc>
+ </method>
+ <field name="LOG" type="org.apache.commons.logging.Log"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="NUM_INPUT_FILES" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="INPUT_DIR_RECURSIVE" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[A base class for file-based {@link InputFormat}.
+
+ <p><code>FileInputFormat</code> is the base class for all file-based
+ <code>InputFormat</code>s. This provides a generic implementation of
+ {@link #getSplits(JobConf, int)}.
+ Subclasses of <code>FileInputFormat</code> can also override the
+ {@link #isSplitable(FileSystem, Path)} method to ensure input-files are
+ not split-up and are processed as a whole by {@link Mapper}s.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileInputFormat -->
+ <!-- start class org.apache.hadoop.mapred.FileInputFormat.Counter -->
+ <class name="FileInputFormat.Counter" extends="java.lang.Enum"
+ abstract="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.mapred.FileInputFormat.Counter[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.mapred.FileInputFormat.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileInputFormat.Counter -->
+ <!-- start class org.apache.hadoop.mapred.FileOutputCommitter -->
+ <class name="FileOutputCommitter" extends="org.apache.hadoop.mapred.OutputCommitter"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="FileOutputCommitter"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getTaskAttemptPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getWorkPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <param name="outputPath" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setupJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="commitJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="cleanupJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="abortJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <param name="runState" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setupTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="commitTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="abortTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="needsTaskCommit" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="isRecoverySupported" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="isRecoverySupported" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.JobContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="recoverTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="context" type="org.apache.hadoop.mapred.TaskAttemptContext"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <field name="LOG" type="org.apache.commons.logging.Log"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="TEMP_DIR_NAME" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Temporary directory name]]>
+ </doc>
+ </field>
+ <field name="SUCCEEDED_FILE_NAME" type="java.lang.String"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[An {@link OutputCommitter} that commits files specified
+ in job output directory i.e. ${mapreduce.output.fileoutputformat.outputdir}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileOutputCommitter -->
+ <!-- start class org.apache.hadoop.mapred.FileOutputFormat -->
+ <class name="FileOutputFormat" extends="java.lang.Object"
+ abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapred.OutputFormat"/>
+ <constructor name="FileOutputFormat"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setCompressOutput"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="compress" type="boolean"/>
+ <doc>
+ <![CDATA[Set whether the output of the job is compressed.
+ @param conf the {@link JobConf} to modify
+ @param compress should the output of the job be compressed?]]>
+ </doc>
+ </method>
+ <method name="getCompressOutput" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Is the job output compressed?
+ @param conf the {@link JobConf} to look in
+ @return <code>true</code> if the job output should be compressed,
+ <code>false</code> otherwise]]>
+ </doc>
+ </method>
+ <method name="setOutputCompressorClass"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="codecClass" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Set the {@link CompressionCodec} to be used to compress job outputs.
+ @param conf the {@link JobConf} to modify
+ @param codecClass the {@link CompressionCodec} to be used to
+ compress the job outputs]]>
+ </doc>
+ </method>
+ <method name="getOutputCompressorClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="defaultValue" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the {@link CompressionCodec} for compressing the job outputs.
+ @param conf the {@link JobConf} to look in
+ @param defaultValue the {@link CompressionCodec} to return if not set
+ @return the {@link CompressionCodec} to be used to compress the
+ job outputs
+ @throws IllegalArgumentException if the class was specified, but not found]]>
+ </doc>
+ </method>
+ <method name="getRecordWriter" return="org.apache.hadoop.mapred.RecordWriter"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="name" type="java.lang.String"/>
+ <param name="progress" type="org.apache.hadoop.util.Progressable"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="checkOutputSpecs"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ignored" type="org.apache.hadoop.fs.FileSystem"/>
+ <param name="job" type="org.apache.hadoop.mapred.JobConf"/>
+ <exception name="FileAlreadyExistsException" type="org.apache.hadoop.mapred.FileAlreadyExistsException"/>
+ <exception name="InvalidJobConfException" type="org.apache.hadoop.mapred.InvalidJobConfException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setOutputPath"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Set the {@link Path} of the output directory for the map-reduce job.
+
+ @param conf The configuration of the job.
+ @param outputDir the {@link Path} of the output directory for
+ the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="setWorkOutputPath"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="outputDir" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Set the {@link Path} of the task's temporary output directory
+ for the map-reduce job.
+
+ <p><i>Note</i>: Task output path is set by the framework.
+ </p>
+ @param conf The configuration of the job.
+ @param outputDir the {@link Path} of the output directory
+ for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="getOutputPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Get the {@link Path} to the output directory for the map-reduce job.
+
+ @return the {@link Path} to the output directory for the map-reduce job.
+ @see FileOutputFormat#getWorkOutputPath(JobConf)]]>
+ </doc>
+ </method>
+ <method name="getWorkOutputPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <doc>
+ <![CDATA[Get the {@link Path} to the task's temporary output directory
+ for the map-reduce job
+
+ <h4 id="SideEffectFiles">Tasks' Side-Effect Files</h4>
+
+ <p><i>Note:</i> The following is valid only if the {@link OutputCommitter}
+ is {@link FileOutputCommitter}. If <code>OutputCommitter</code> is not
+ a <code>FileOutputCommitter</code>, the task's temporary output
+ directory is same as {@link #getOutputPath(JobConf)} i.e.
+ <tt>${mapreduce.output.fileoutputformat.outputdir}$</tt></p>
+
+ <p>Some applications need to create/write-to side-files, which differ from
+ the actual job-outputs.
+
+ <p>In such cases there could be issues with 2 instances of the same TIP
+ (running simultaneously e.g. speculative tasks) trying to open/write-to the
+ same file (path) on HDFS. Hence the application-writer will have to pick
+ unique names per task-attempt (e.g. using the attemptid, say
+ <tt>attempt_200709221812_0001_m_000000_0</tt>), not just per TIP.</p>
+
+ <p>To get around this the Map-Reduce framework helps the application-writer
+ out by maintaining a special
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt>
+ sub-directory for each task-attempt on HDFS where the output of the
+ task-attempt goes. On successful completion of the task-attempt the files
+ in the <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}</tt> (only)
+ are <i>promoted</i> to <tt>${mapreduce.output.fileoutputformat.outputdir}</tt>. Of course, the
+ framework discards the sub-directory of unsuccessful task-attempts. This
+ is completely transparent to the application.</p>
+
+ <p>The application-writer can take advantage of this by creating any
+ side-files required in <tt>${mapreduce.task.output.dir}</tt> during execution
+ of his reduce-task i.e. via {@link #getWorkOutputPath(JobConf)}, and the
+ framework will move them out similarly - thus she doesn't have to pick
+ unique paths per task-attempt.</p>
+
+ <p><i>Note</i>: the value of <tt>${mapreduce.task.output.dir}</tt> during
+ execution of a particular task-attempt is actually
+ <tt>${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}</tt>, and this value is
+ set by the map-reduce framework. So, just create any side-files in the
+ path returned by {@link #getWorkOutputPath(JobConf)} from map/reduce
+ task to take advantage of this feature.</p>
+
+ <p>The entire discussion holds true for maps of jobs with
+ reducer=NONE (i.e. 0 reduces) since output of the map, in that case,
+ goes directly to HDFS.</p>
+
+ @return the {@link Path} to the task's temporary output directory
+ for the map-reduce job.]]>
+ </doc>
+ </method>
+ <method name="getTaskOutputPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="name" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Helper function to create the task's temporary output directory and
+ return the path to the task's output file.
+
+ @param conf job-configuration
+ @param name temporary task-output filename
+ @return path to the task's temporary output file
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getUniqueName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Helper function to generate a name that is unique for the task.
+
+ <p>The generated name can be used to create custom files from within the
+ different tasks for the job, the names for different tasks will not collide
+ with each other.</p>
+
+ <p>The given name is postfixed with the task type, 'm' for maps, 'r' for
+ reduces and the task partition number. For example, give a name 'test'
+ running on the first map o the job the generated name will be
+ 'test-m-00000'.</p>
+
+ @param conf the configuration for the job.
+ @param name the name to make unique.
+ @return a unique name accross all tasks of the job.]]>
+ </doc>
+ </method>
+ <method name="getPathForCustomFile" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.mapred.JobConf"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Helper function to generate a {@link Path} for a file that is unique for
+ the task within the job output directory.
+
+ <p>The path can be used to create custom files from within the map and
+ reduce tasks. The path name will be unique for each task. The path parent
+ will be the job output directory.</p>ls
+
+ <p>This method uses the {@link #getUniqueName} method to make the file name
+ unique for the task.</p>
+
+ @param conf the configuration for the job.
+ @param name the name for the file.
+ @return a unique path accross all tasks of the job.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[A base class for {@link OutputFormat}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileOutputFormat -->
+ <!-- start class org.apache.hadoop.mapred.FileOutputFormat.Counter -->
+ <class name="FileOutputFormat.Counter" extends="java.lang.Enum"
+ abstract="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.mapred.FileOutputFormat.Counter[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.mapred.FileOutputFormat.Counter"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.FileOutputFormat.Counter -->
+ <!-- start class org.apache.hadoop.mapred.FileSplit -->
+ <class name="FileSplit" extends="org.apache.hadoop.mapreduce.InputSplit"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapred.InputSplitWithLocationInfo"/>
+ <constructor name="FileSplit"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="FileSplit" type="org.apache.hadoop.fs.Path, long, long, org.apache.hadoop.mapred.JobConf"
+ static="false" final="false" visibility="public"
+ deprecated="deprecated, no comm
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-10764. Fix
INodeFile#getBlocks to not return null. Contributed by Arpit Agarwal.
Posted by cd...@apache.org.
HDFS-10764. Fix INodeFile#getBlocks to not return null. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0faee62a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0faee62a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0faee62a
Branch: refs/heads/HDFS-9806
Commit: 0faee62a0c8c1b8fd83227babfd00fbc2b26bddf
Parents: 99603e9
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Aug 19 22:13:36 2016 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Aug 19 22:13:36 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/INodeFile.java | 29 ++++++++------------
1 file changed, 11 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0faee62a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 63945a4..12ead7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -200,8 +200,8 @@ public class INodeFile extends INodeWithAdditionalFields
public INodeFile(INodeFile that) {
super(that);
this.header = that.header;
- this.blocks = that.blocks;
this.features = that.features;
+ setBlocks(that.blocks);
}
public INodeFile(INodeFile that, FileDiffList diffs) {
@@ -271,9 +271,6 @@ public class INodeFile extends INodeWithAdditionalFields
/** Assert all blocks are complete. */
private void assertAllBlocksComplete(int numCommittedAllowed,
short minReplication) {
- if (blocks == null) {
- return;
- }
for (int i = 0; i < blocks.length; i++) {
final String err = checkBlockComplete(blocks, i, numCommittedAllowed,
minReplication);
@@ -342,7 +339,7 @@ public class INodeFile extends INodeWithAdditionalFields
BlockInfo removeLastBlock(Block oldblock) {
Preconditions.checkState(isUnderConstruction(),
"file is no longer under construction");
- if (blocks == null || blocks.length == 0) {
+ if (blocks.length == 0) {
return null;
}
int size_1 = blocks.length - 1;
@@ -618,7 +615,7 @@ public class INodeFile extends INodeWithAdditionalFields
*/
void addBlock(BlockInfo newblock) {
Preconditions.checkArgument(newblock.isStriped() == this.isStriped());
- if (this.blocks == null) {
+ if (this.blocks.length == 0) {
this.setBlocks(new BlockInfo[]{newblock});
} else {
int size = this.blocks.length;
@@ -631,12 +628,12 @@ public class INodeFile extends INodeWithAdditionalFields
/** Set the blocks. */
private void setBlocks(BlockInfo[] blocks) {
- this.blocks = blocks;
+ this.blocks = (blocks != null ? blocks : BlockInfo.EMPTY_ARRAY);
}
/** Clear all blocks of the file. */
public void clearBlocks() {
- setBlocks(BlockInfo.EMPTY_ARRAY);
+ this.blocks = BlockInfo.EMPTY_ARRAY;
}
@Override
@@ -836,7 +833,7 @@ public class INodeFile extends INodeWithAdditionalFields
*/
public final long computeFileSize(boolean includesLastUcBlock,
boolean usePreferredBlockSize4LastUcBlock) {
- if (blocks == null || blocks.length == 0) {
+ if (blocks.length == 0) {
return 0;
}
final int last = blocks.length - 1;
@@ -876,10 +873,6 @@ public class INodeFile extends INodeWithAdditionalFields
// TODO: support EC with heterogeneous storage
public final QuotaCounts storagespaceConsumedStriped() {
QuotaCounts counts = new QuotaCounts.Builder().build();
- if (blocks == null || blocks.length == 0) {
- return counts;
- }
-
for (BlockInfo b : blocks) {
Preconditions.checkState(b.isStriped());
long blockSize = b.isComplete() ?
@@ -931,7 +924,7 @@ public class INodeFile extends INodeWithAdditionalFields
* Return the penultimate allocated block for this file.
*/
BlockInfo getPenultimateBlock() {
- if (blocks == null || blocks.length <= 1) {
+ if (blocks.length <= 1) {
return null;
}
return blocks[blocks.length - 2];
@@ -939,12 +932,12 @@ public class INodeFile extends INodeWithAdditionalFields
@Override
public BlockInfo getLastBlock() {
- return blocks == null || blocks.length == 0? null: blocks[blocks.length-1];
+ return blocks.length == 0 ? null: blocks[blocks.length-1];
}
@Override
public int numBlocks() {
- return blocks == null ? 0 : blocks.length;
+ return blocks.length;
}
@VisibleForTesting
@@ -955,7 +948,7 @@ public class INodeFile extends INodeWithAdditionalFields
out.print(", fileSize=" + computeFileSize(snapshotId));
// only compare the first block
out.print(", blocks=");
- out.print(blocks == null || blocks.length == 0? null: blocks[0]);
+ out.print(blocks.length == 0 ? null: blocks[0]);
out.println();
}
@@ -1004,7 +997,7 @@ public class INodeFile extends INodeWithAdditionalFields
long newLength, BlockStoragePolicy bsps,
QuotaCounts delta) {
final BlockInfo[] blocks = getBlocks();
- if (blocks == null || blocks.length == 0) {
+ if (blocks.length == 0) {
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HADOOP-13497. fix wrong command in
CredentialProviderAPI.md. Contributed by Yuanbo Liu.
Posted by cd...@apache.org.
HADOOP-13497. fix wrong command in CredentialProviderAPI.md. Contributed by Yuanbo Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aae8d6b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aae8d6b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aae8d6b
Branch: refs/heads/HDFS-9806
Commit: 8aae8d6bf03ade0607547ed461dc99a336a7e9d4
Parents: 126d165
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Wed Aug 24 00:03:27 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Wed Aug 24 00:03:27 2016 +0900
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/CredentialProviderAPI.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aae8d6b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
index 1142372..a40bf2b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
@@ -119,7 +119,7 @@ See the command options detail in the [Commands Manual](CommandsManual.html#cred
Utilizing the credential command will often be for provisioning a password or secret to a particular credential store provider. In order to explicitly indicate which provider store to use the `-provider` option should be used.
-Example: `hadoop credential create ssl.server.keystore.password jceks://file/tmp/test.jceks`
+Example: `hadoop credential create ssl.server.keystore.password -provider jceks://file/tmp/test.jceks`
In order to indicate a particular provider type and location, the user must provide the `hadoop.security.credential.provider.path` configuration element in core-site.xml or use the command line option `-provider` on each of the credential management commands. This provider path is a comma-separated list of URLs that indicates the type and location of a list of providers that should be consulted. For example, the following path: `user:///,jceks://file/tmp/test.jceks,jceks://hdfs@nn1.example.com/my/path/test.jceks` indicates that the current user's credentials file should be consulted through the User Provider, that the local file located at `/tmp/test.jceks` is a Java Keystore Provider and that the file located within HDFS at `nn1.example.com/my/path/test.jceks` is also a store for a Java Keystore Provider.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HADOOP-13512.
ReloadingX509TrustManager should keep reloading in case of exception.
(Contributed by Mingliang Liu)
Posted by cd...@apache.org.
HADOOP-13512. ReloadingX509TrustManager should keep reloading in case of exception. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f51eae0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f51eae0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f51eae0
Branch: refs/heads/HDFS-9806
Commit: 0f51eae0c085ded38216824377acf8122638c3a5
Parents: ae4db25
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Aug 18 00:04:58 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Aug 18 14:10:37 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f51eae0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java
index bb90a61..597f8d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java
@@ -165,10 +165,10 @@ public final class ReloadingX509TrustManager
throws IOException, GeneralSecurityException {
X509TrustManager trustManager = null;
KeyStore ks = KeyStore.getInstance(type);
- lastLoaded = file.lastModified();
FileInputStream in = new FileInputStream(file);
try {
ks.load(in, password.toCharArray());
+ lastLoaded = file.lastModified();
LOG.debug("Loaded truststore '" + file + "'");
} finally {
in.close();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: MAPREDUCE-6762.
ControlledJob#toString failed with NPE when job status is not successfully
updated (Weiwei Yang via Varun Saxena)
Posted by cd...@apache.org.
MAPREDUCE-6762. ControlledJob#toString failed with NPE when job status is not successfully updated (Weiwei Yang via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d37b45d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d37b45d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d37b45d6
Branch: refs/heads/HDFS-9806
Commit: d37b45d613b768950d1cbe342961cd71776816ae
Parents: 0faee62
Author: Varun Saxena <va...@apache.org>
Authored: Sun Aug 21 21:46:17 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Sun Aug 21 21:46:17 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapreduce/Job.java | 2 +-
.../org/apache/hadoop/mapreduce/TestJob.java | 36 ++++++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37b45d6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
index 33e820b..45c065d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
@@ -422,7 +422,7 @@ public class Job extends JobContextImpl implements JobContext {
* The user-specified job name.
*/
public String getJobName() {
- if (state == JobState.DEFINE) {
+ if (state == JobState.DEFINE || status == null) {
return super.getJobName();
}
ensureState(JobState.RUNNING);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37b45d6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
index 71bacf7..60f390f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.JobStatus.State;
+import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -55,6 +56,41 @@ public class TestJob {
}
@Test
+ public void testUnexpectedJobStatus() throws Exception {
+ Cluster cluster = mock(Cluster.class);
+ JobID jobid = new JobID("1014873536921", 6);
+ ClientProtocol clientProtocol = mock(ClientProtocol.class);
+ when(cluster.getClient()).thenReturn(clientProtocol);
+ JobStatus status = new JobStatus(jobid, 0f, 0f, 0f, 0f,
+ State.RUNNING, JobPriority.DEFAULT, "root",
+ "testUnexpectedJobStatus", "job file", "tracking URL");
+ when(clientProtocol.getJobStatus(jobid)).thenReturn(status);
+ Job job = Job.getInstance(cluster, status, new JobConf());
+
+ // ensurer job status is RUNNING
+ Assert.assertNotNull(job.getStatus());
+ Assert.assertTrue(job.getStatus().getState() == State.RUNNING);
+
+ // when updating job status, job client could not retrieve
+ // job status, and status reset to null
+ when(clientProtocol.getJobStatus(jobid)).thenReturn(null);
+
+ try {
+ job.updateStatus();
+ } catch (IOException e) {
+ Assert.assertTrue(e != null
+ && e.getMessage().contains("Job status not available"));
+ }
+
+ try {
+ ControlledJob cj = new ControlledJob(job, null);
+ Assert.assertNotNull(cj.toString());
+ } catch (NullPointerException e) {
+ Assert.fail("job API fails with NPE");
+ }
+ }
+
+ @Test
public void testUGICredentialsPropogation() throws Exception {
Credentials creds = new Credentials();
Token<?> token = mock(Token.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HADOOP-13524. mvn eclipse:eclipse
generates .gitignore'able files. Contributed by Vinod Kumar Vavilapalli
Posted by cd...@apache.org.
HADOOP-13524. mvn eclipse:eclipse generates .gitignore'able files. Contributed by Vinod Kumar Vavilapalli
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd76238a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd76238a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd76238a
Branch: refs/heads/HDFS-9806
Commit: dd76238a3bafd58faa6f38f075505bef1012f150
Parents: e90f335
Author: Jian He <ji...@apache.org>
Authored: Tue Aug 23 19:13:14 2016 +0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Aug 23 19:13:14 2016 +0800
----------------------------------------------------------------------
.gitignore | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd76238a/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 998287d..a5d69d0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,11 @@
.settings
target
build
+
+# External tool builders
+*/.externalToolBuilders
+*/maven-eclipse.xml
+
hadoop-common-project/hadoop-kms/downloads/
hadoop-hdfs-project/hadoop-hdfs/downloads
hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-13526. Add detailed logging
in KMS for the authentication failure of proxy user. Contributed by Suraj
Acharya.
Posted by cd...@apache.org.
HADOOP-13526. Add detailed logging in KMS for the authentication failure of proxy user. Contributed by Suraj Acharya.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4070caad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4070caad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4070caad
Branch: refs/heads/HDFS-9806
Commit: 4070caad70db49b50554088d29ac2fbc7ba62a0a
Parents: 3ca4d6d
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Aug 22 18:06:53 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Aug 22 18:09:35 2016 -0700
----------------------------------------------------------------------
.../web/DelegationTokenAuthenticationFilter.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4070caad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index fb6817e..112c952 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.util.HttpExceptionUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
@@ -81,6 +83,9 @@ public class DelegationTokenAuthenticationFilter
private static final String ERROR_EXCEPTION_JSON = "exception";
private static final String ERROR_MESSAGE_JSON = "message";
+ private static final Logger LOG = LoggerFactory.getLogger(
+ DelegationTokenAuthenticationFilter.class);
+
/**
* Sets an external <code>DelegationTokenSecretManager</code> instance to
* manage creation and verification of Delegation Tokens.
@@ -261,6 +266,11 @@ public class DelegationTokenAuthenticationFilter
HttpExceptionUtils.createServletExceptionResponse(response,
HttpServletResponse.SC_FORBIDDEN, ex);
requestCompleted = true;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Authentication exception: " + ex.getMessage(), ex);
+ } else {
+ LOG.warn("Authentication exception: " + ex.getMessage());
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-4491. yarn list command to
support filtering by tags. Contributed by Varun Saxena
Posted by cd...@apache.org.
YARN-4491. yarn list command to support filtering by tags. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/143c59e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/143c59e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/143c59e4
Branch: refs/heads/HDFS-9806
Commit: 143c59e4c5a811eb2c12cf6626d558f9b8796e03
Parents: 8aae8d6
Author: Naganarasimha <na...@apache.org>
Authored: Wed Aug 24 01:53:02 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Wed Aug 24 01:53:02 2016 +0530
----------------------------------------------------------------------
.../hadoop/mapred/ResourceMgrDelegate.java | 10 +
.../hadoop/yarn/client/api/YarnClient.java | 25 ++
.../yarn/client/api/impl/YarnClientImpl.java | 11 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 42 +++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 241 ++++++++++++++++---
5 files changed, 282 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index cc164fd..159b518 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -349,6 +349,16 @@ public class ResourceMgrDelegate extends YarnClient {
}
@Override
+ public List<ApplicationReport> getApplications(
+ Set<String> applicationTypes,
+ EnumSet<YarnApplicationState> applicationStates,
+ Set<String> applicationTags)
+ throws YarnException, IOException {
+ return client.getApplications(
+ applicationTypes, applicationStates, applicationTags);
+ }
+
+ @Override
public List<ApplicationReport> getApplications(Set<String> queues,
Set<String> users, Set<String> applicationTypes,
EnumSet<YarnApplicationState> applicationStates) throws YarnException,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 218bb34..619ea0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -314,6 +314,31 @@ public abstract class YarnClient extends AbstractService {
/**
* <p>
+ * Get a report (ApplicationReport) of Applications matching the given
+ * application types, application states and application tags in the cluster.
+ * </p>
+ *
+ * <p>
+ * If the user does not have <code>VIEW_APP</code> access for an application
+ * then the corresponding report will be filtered as described in
+ * {@link #getApplicationReport(ApplicationId)}.
+ * </p>
+ *
+ * @param applicationTypes set of application types you are interested in
+ * @param applicationStates set of application states you are interested in
+ * @param applicationTags set of application tags you are interested in
+ * @return a list of reports of applications
+ * @throws YarnException
+ * @throws IOException
+ */
+ public abstract List<ApplicationReport> getApplications(
+ Set<String> applicationTypes,
+ EnumSet<YarnApplicationState> applicationStates,
+ Set<String> applicationTags) throws YarnException,
+ IOException;
+
+ /**
+ * <p>
* Get a report (ApplicationReport) of Applications matching the given users,
* queues, application types and application states in the cluster. If any of
* the params is set to null, it is not used when filtering.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index dee31c5..b91fde0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -505,6 +505,17 @@ public class YarnClientImpl extends YarnClient {
}
@Override
+ public List<ApplicationReport> getApplications(Set<String> applicationTypes,
+ EnumSet<YarnApplicationState> applicationStates,
+ Set<String> applicationTags) throws YarnException, IOException {
+ GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
+ request.setApplicationTags(applicationTags);
+ GetApplicationsResponse response = rmClient.getApplications(request);
+ return response.getApplicationList();
+ }
+
+ @Override
public List<ApplicationReport> getApplications(Set<String> queues,
Set<String> users, Set<String> applicationTypes,
EnumSet<YarnApplicationState> applicationStates) throws YarnException,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 865ce00..f0b1c47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -73,6 +73,7 @@ public class ApplicationCLI extends YarnCLI {
private static final String APP_TYPE_CMD = "appTypes";
private static final String APP_STATE_CMD = "appStates";
+ private static final String APP_TAG_CMD = "appTags";
private static final String ALLSTATES_OPTION = "ALL";
private static final String QUEUE_CMD = "queue";
public static final String APPLICATION = "application";
@@ -102,8 +103,9 @@ public class ApplicationCLI extends YarnCLI {
"Prints the status of the application.");
opts.addOption(LIST_CMD, false, "List applications. "
+ "Supports optional use of -appTypes to filter applications "
- + "based on application type, "
- + "and -appStates to filter applications based on application state.");
+ + "based on application type, -appStates to filter applications "
+ + "based on application state and -appTags to filter applications "
+ + "based on application tag.");
opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a "
+ "different queue.");
opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to"
@@ -123,6 +125,13 @@ public class ApplicationCLI extends YarnCLI {
appStateOpt.setArgs(Option.UNLIMITED_VALUES);
appStateOpt.setArgName("States");
opts.addOption(appStateOpt);
+ Option appTagOpt = new Option(APP_TAG_CMD, true, "Works with -list to "
+ + "filter applications based on input comma-separated list of "
+ + "application tags.");
+ appTagOpt.setValueSeparator(',');
+ appTagOpt.setArgs(Option.UNLIMITED_VALUES);
+ appTagOpt.setArgName("Tags");
+ opts.addOption(appTagOpt);
opts.addOption(APP_ID, true, "Specify Application Id to be operated");
opts.addOption(UPDATE_PRIORITY, true,
"update priority of an application. ApplicationId can be"
@@ -229,7 +238,19 @@ public class ApplicationCLI extends YarnCLI {
}
}
}
- listApplications(appTypes, appStates);
+
+ Set<String> appTags = new HashSet<String>();
+ if (cliParser.hasOption(APP_TAG_CMD)) {
+ String[] tags = cliParser.getOptionValues(APP_TAG_CMD);
+ if (tags != null) {
+ for (String tag : tags) {
+ if (!tag.trim().isEmpty()) {
+ appTags.add(tag.trim());
+ }
+ }
+ }
+ }
+ listApplications(appTypes, appStates, appTags);
} else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
if (args.length != 3) {
printUsage(title, opts);
@@ -434,17 +455,18 @@ public class ApplicationCLI extends YarnCLI {
}
/**
- * Lists the applications matching the given application Types And application
- * States present in the Resource Manager
+ * Lists the applications matching the given application Types, application
+ * States and application Tags present in the Resource Manager.
*
* @param appTypes
* @param appStates
+ * @param appTags
* @throws YarnException
* @throws IOException
*/
private void listApplications(Set<String> appTypes,
- EnumSet<YarnApplicationState> appStates) throws YarnException,
- IOException {
+ EnumSet<YarnApplicationState> appStates, Set<String> appTags)
+ throws YarnException, IOException {
PrintWriter writer = new PrintWriter(
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
if (allAppStates) {
@@ -460,11 +482,11 @@ public class ApplicationCLI extends YarnCLI {
}
List<ApplicationReport> appsReport = client.getApplications(appTypes,
- appStates);
+ appStates, appTags);
writer.println("Total number of applications (application-types: "
- + appTypes + " and states: " + appStates + ")" + ":"
- + appsReport.size());
+ + appTypes + ", states: " + appStates + " and tags: " + appTags + ")"
+ + ":" + appsReport.size());
writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name",
"Application-Type", "User", "Queue", "State", "Final-State",
"Progress", "Tracking-URL");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 6d7aaa7..1cd513b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -87,6 +87,7 @@ import org.junit.Test;
import org.mortbay.log.Log;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
public class TestYarnCLI {
@@ -373,8 +374,10 @@ public class TestYarnCLI {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
- List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ Sets.newHashSet("tag1", "tag3"), false, Priority.UNDEFINED, "", "");
+ List<ApplicationReport> applicationReports =
+ new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
@@ -383,7 +386,8 @@ public class TestYarnCLI {
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
- null);
+ null, Sets.newHashSet("tag2", "tag3"), false, Priority.UNDEFINED,
+ "", "");
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -392,7 +396,8 @@ public class TestYarnCLI {
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
- null);
+ null, Sets.newHashSet("tag1", "tag4"), false, Priority.UNDEFINED,
+ "", "");
applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -400,8 +405,9 @@ public class TestYarnCLI {
applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4),
"user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
- null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
+ "NON-MAPREDUCE", null, Sets.newHashSet("tag1"), false,
+ Priority.UNDEFINED, "", "");
applicationReports.add(newApplicationReport4);
ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
@@ -409,8 +415,8 @@ public class TestYarnCLI {
applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5),
"user5", "queue5", "appname5", "host5", 128, null,
YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5,
- FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE",
- null);
+ FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", null,
+ Sets.newHashSet("tag2", "tag4"), false, Priority.UNDEFINED, "", "");
applicationReports.add(newApplicationReport5);
ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
@@ -419,7 +425,7 @@ public class TestYarnCLI {
"user6", "queue6", "appname6", "host6", 129, null,
YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6,
FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG",
- null);
+ null, new HashSet<String>(), false, Priority.UNDEFINED, "", "");
applicationReports.add(newApplicationReport6);
// Test command yarn application -list
@@ -433,16 +439,18 @@ public class TestYarnCLI {
appState1.add(YarnApplicationState.RUNNING);
appState1.add(YarnApplicationState.ACCEPTED);
appState1.add(YarnApplicationState.SUBMITTED);
- when(client.getApplications(appType1, appState1)).thenReturn(
- getApplicationReports(applicationReports, appType1, appState1, false));
+ Set<String> appTag = new HashSet<String>();
+ when(client.getApplications(appType1, appState1, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType1, appState1, appTag, false));
int result = cli.run(new String[] { "application", "-list" });
assertEquals(0, result);
- verify(client).getApplications(appType1, appState1);
+ verify(client).getApplications(appType1, appState1, appTag);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType1
- + " and states: " + appState1 + ")" + ":" + 4);
+ + ", states: " + appState1 + " and tags: " + appTag + ")" + ":" + 4);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -487,17 +495,18 @@ public class TestYarnCLI {
appState2.add(YarnApplicationState.RUNNING);
appState2.add(YarnApplicationState.ACCEPTED);
appState2.add(YarnApplicationState.SUBMITTED);
- when(client.getApplications(appType2, appState2)).thenReturn(
- getApplicationReports(applicationReports, appType2, appState2, false));
+ when(client.getApplications(appType2, appState2, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType2, appState2, appTag, false));
result =
cli.run(new String[] { "application", "-list", "-appTypes",
"YARN, ,, NON-YARN", " ,, ,," });
assertEquals(0, result);
- verify(client).getApplications(appType2, appState2);
+ verify(client).getApplications(appType2, appState2, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType2
- + " and states: " + appState2 + ")" + ":" + 1);
+ + ", states: " + appState2 + " and tags: " + appTag + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -522,17 +531,18 @@ public class TestYarnCLI {
appState3.add(YarnApplicationState.FINISHED);
appState3.add(YarnApplicationState.FAILED);
- when(client.getApplications(appType3, appState3)).thenReturn(
- getApplicationReports(applicationReports, appType3, appState3, false));
+ when(client.getApplications(appType3, appState3, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType3, appState3, appTag, false));
result =
cli.run(new String[] { "application", "-list", "--appStates",
"FINISHED ,, , FAILED", ",,FINISHED" });
assertEquals(0, result);
- verify(client).getApplications(appType3, appState3);
+ verify(client).getApplications(appType3, appState3, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType3
- + " and states: " + appState3 + ")" + ":" + 2);
+ + ", states: " + appState3 + " and tags: " + appTag + ")" + ":" + 2);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -565,17 +575,18 @@ public class TestYarnCLI {
appState4.add(YarnApplicationState.FINISHED);
appState4.add(YarnApplicationState.FAILED);
- when(client.getApplications(appType4, appState4)).thenReturn(
- getApplicationReports(applicationReports, appType4, appState4, false));
+ when(client.getApplications(appType4, appState4, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType4, appState4, appTag, false));
result =
cli.run(new String[] { "application", "-list", "--appTypes",
"YARN,NON-YARN", "--appStates", "FINISHED ,, , FAILED" });
assertEquals(0, result);
- verify(client).getApplications(appType2, appState2);
+ verify(client).getApplications(appType2, appState2, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType4
- + " and states: " + appState4 + ")" + ":" + 1);
+ + ", states: " + appState4 + " and tags: " + appTag + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -620,17 +631,18 @@ public class TestYarnCLI {
EnumSet<YarnApplicationState> appState5 =
EnumSet.noneOf(YarnApplicationState.class);
appState5.add(YarnApplicationState.FINISHED);
- when(client.getApplications(appType5, appState5)).thenReturn(
- getApplicationReports(applicationReports, appType5, appState5, true));
+ when(client.getApplications(appType5, appState5, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType5, appState5, appTag, true));
result =
cli.run(new String[] { "application", "-list", "--appStates",
"FINISHED ,, , ALL" });
assertEquals(0, result);
- verify(client).getApplications(appType5, appState5);
+ verify(client).getApplications(appType5, appState5, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType5
- + " and states: " + appState5 + ")" + ":" + 6);
+ + ", states: " + appState5 + " and tags: " + appTag + ")" + ":" + 6);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -680,17 +692,18 @@ public class TestYarnCLI {
EnumSet<YarnApplicationState> appState6 =
EnumSet.noneOf(YarnApplicationState.class);
appState6.add(YarnApplicationState.FINISHED);
- when(client.getApplications(appType6, appState6)).thenReturn(
- getApplicationReports(applicationReports, appType6, appState6, false));
+ when(client.getApplications(appType6, appState6, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType6, appState6, appTag, false));
result =
cli.run(new String[] { "application", "-list", "-appTypes",
"YARN, ,, NON-YARN", "--appStates", "finished" });
assertEquals(0, result);
- verify(client).getApplications(appType6, appState6);
+ verify(client).getApplications(appType6, appState6, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType6
- + " and states: " + appState6 + ")" + ":" + 1);
+ + ", states: " + appState6 + " and tags: " + appTag + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -705,12 +718,146 @@ public class TestYarnCLI {
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
+
+ // Test command yarn application with tags.
+ sysOutStream.reset();
+ Set<String> appTag1 = Sets.newHashSet("tag1");
+ when(client.getApplications(appType1, appState1, appTag1)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType1, appState1, appTag1, false));
+ result =
+ cli.run(new String[] { "application", "-list", "-appTags", "tag1" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType1, appState1, appTag1);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType1
+ + ", states: " + appState1 + " and tags: " + appTag1 + ")" + ":" + 2);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0005\t ");
+ pw.print("appname\t YARN\t user\t ");
+ pw.print("queue\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 53.79%");
+ pw.println("\t N/A");
+ pw.print(" application_1234_0007\t ");
+ pw.print("appname3\t MAPREDUCE\t user3\t ");
+ pw.print("queue3\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 73.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ EnumSet<YarnApplicationState> appState7 =
+ EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.FAILED);
+ when(client.getApplications(appType1, appState7, appTag1)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType1, appState7, appTag1, false));
+ result = cli.run(
+ new String[] { "application", "-list", "-appStates", "RUNNING,FAILED",
+ "-appTags", "tag1" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType1, appState7, appTag1);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType1
+ + ", states: " + appState7 + " and tags: " + appTag1 + ")" + ":" + 3);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0005\t ");
+ pw.print("appname\t YARN\t user\t ");
+ pw.print("queue\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 53.79%");
+ pw.println("\t N/A");
+ pw.print(" application_1234_0007\t ");
+ pw.print("appname3\t MAPREDUCE\t user3\t ");
+ pw.print("queue3\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 73.79%");
+ pw.println("\t N/A");
+ pw.print(" application_1234_0008\t ");
+ pw.print("appname4\t NON-MAPREDUCE\t user4\t ");
+ pw.print("queue4\t FAILED\t ");
+ pw.print("SUCCEEDED\t 83.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ Set<String> appType9 = Sets.newHashSet("YARN");
+ Set<String> appTag2 = Sets.newHashSet("tag3");
+ when(client.getApplications(appType9, appState1, appTag2)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType9, appState1, appTag2, false));
+ result = cli.run(new String[] { "application", "-list", "-appTypes", "YARN",
+ "-appTags", "tag3" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType9, appState1, appTag2);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType9
+ + ", states: " + appState1 + " and tags: " + appTag2 + ")" + ":" + 1);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0005\t ");
+ pw.print("appname\t YARN\t user\t ");
+ pw.print("queue\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 53.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ Set<String> appType10 = Sets.newHashSet("HIVE");
+ Set<String> appTag3 = Sets.newHashSet("tag4");
+ EnumSet<YarnApplicationState> appState10 =
+ EnumSet.of(YarnApplicationState.ACCEPTED);
+ when(client.getApplications(appType10, appState10, appTag3)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType10, appState10, appTag3, false));
+ result = cli.run(new String[] { "application", "-list", "-appTypes", "HIVE",
+ "-appStates", "ACCEPTED", "-appTags", "tag4" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType10, appState10, appTag3);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType10
+ + ", states: " + appState10 + " and tags: " + appTag3 + ")" + ":" + 1);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0009\t ");
+ pw.print("appname5\t HIVE\t user5\t ");
+ pw.print("queue5\t ACCEPTED\t ");
+ pw.print("KILLED\t 93.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(10)).write(any(byte[].class), anyInt(), anyInt());
}
private List<ApplicationReport> getApplicationReports(
List<ApplicationReport> applicationReports,
Set<String> appTypes, EnumSet<YarnApplicationState> appStates,
- boolean allStates) {
+ Set<String> appTags, boolean allStates) {
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
@@ -732,6 +879,22 @@ public class TestYarnCLI {
}
}
+ if (appTags != null && !appTags.isEmpty()) {
+ Set<String> tags = appReport.getApplicationTags();
+ if (tags == null || tags.isEmpty()) {
+ continue;
+ }
+ boolean match = false;
+ for (String appTag : appTags) {
+ if (tags.contains(appTag)) {
+ match = true;
+ break;
+ }
+ }
+ if (!match) {
+ continue;
+ }
+ }
appReports.add(appReport);
}
return appReports;
@@ -1816,6 +1979,9 @@ public class TestYarnCLI {
pw.println(" state can be one of the following:");
pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN");
pw.println(" NING,FINISHED,FAILED,KILLED");
+ pw.println(" -appTags <Tags> Works with -list to filter applications");
+ pw.println(" based on input comma-separated list of");
+ pw.println(" application tags.");
pw.println(" -appTypes <Types> Works with -list to filter applications");
pw.println(" based on input comma-separated list of");
pw.println(" application types.");
@@ -1825,9 +1991,10 @@ public class TestYarnCLI {
pw.println(" with space");
pw.println(" -list List applications. Supports optional use");
pw.println(" of -appTypes to filter applications based");
- pw.println(" on application type, and -appStates to");
- pw.println(" filter applications based on application");
- pw.println(" state.");
+ pw.println(" on application type, -appStates to filter");
+ pw.println(" applications based on application state");
+ pw.println(" and -appTags to filter applications based");
+ pw.println(" on application tag.");
pw.println(" -movetoqueue <Application ID> Moves the application to a different");
pw.println(" queue.");
pw.println(" -queue <Queue Name> Works with the movetoqueue command to");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HADOOP-12726. Unsupported FS
operations should throw UnsupportedOperationException. Contributed by Daniel
Templeton.
Posted by cd...@apache.org.
HADOOP-12726. Unsupported FS operations should throw UnsupportedOperationException. Contributed by Daniel Templeton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c37346d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c37346d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c37346d0
Branch: refs/heads/HDFS-9806
Commit: c37346d0e3f9d39d0aec7a9c5bda3e9772aa969b
Parents: 143c59e
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Aug 23 14:05:57 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Aug 23 14:12:52 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/ChecksumFileSystem.java | 6 ++++--
.../src/main/java/org/apache/hadoop/fs/ChecksumFs.java | 3 ++-
.../main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java | 3 ++-
.../main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java | 4 ++--
.../apache/hadoop/metrics2/sink/RollingFileSystemSink.java | 6 +++---
.../hadoop-common/src/site/markdown/filesystem/filesystem.md | 8 ++++----
.../main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 ++-
.../org/apache/hadoop/fs/s3native/NativeS3FileSystem.java | 3 ++-
8 files changed, 21 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index 1f14c4d..e4c0b33 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -356,12 +356,14 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
@Override
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by ChecksumFileSystem");
}
@Override
public boolean truncate(Path f, long newLength) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Truncate is not supported "
+ + "by ChecksumFileSystem");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
index 2b632a1..6e98db5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
@@ -297,7 +297,8 @@ public abstract class ChecksumFs extends FilterFs {
@Override
public boolean truncate(Path f, long newLength) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Truncate is not supported "
+ + "by ChecksumFs");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index d429c42..f1afacd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -288,7 +288,8 @@ public class FTPFileSystem extends FileSystem {
@Override
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by FTPFileSystem");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
index 8b6267a..30cf4d3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
@@ -77,7 +77,6 @@ public class SFTPFileSystem extends FileSystem {
"Can't make directory for path \"%s\" under \"%s\".";
public static final String E_DIR_NOTEMPTY = "Directory: %s is not empty.";
public static final String E_FILE_CHECK_FAILED = "File check failed";
- public static final String E_NOT_SUPPORTED = "Not supported";
public static final String E_SPATH_NOTEXIST = "Source path %s does not exist";
public static final String E_DPATH_EXIST =
"Destination path %s already exist, cannot rename!";
@@ -578,7 +577,8 @@ public class SFTPFileSystem extends FileSystem {
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress)
throws IOException {
- throw new IOException(E_NOT_SUPPORTED);
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by SFTPFileSystem");
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
index 9bfd519..91942f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
@@ -485,10 +485,10 @@ public class RollingFileSystemSink implements MetricsSink, Closeable {
try {
fs.append(basePath);
+ } catch (UnsupportedOperationException ex) {
+ canAppend = false;
} catch (IOException ex) {
- if (ex.getMessage().equals("Not supported")) {
- canAppend = false;
- }
+ // Ignore. The operation is supported.
}
return canAppend;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index 72dbc3c..1587842 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -611,7 +611,7 @@ this precondition fails.
### `FSDataOutputStream append(Path p, int bufferSize, Progressable progress)`
-Implementations MAY throw `UnsupportedOperationException`.
+Implementations without a compliant call SHOULD throw `UnsupportedOperationException`.
#### Preconditions
@@ -630,7 +630,7 @@ by appending data to the existing list.
### `FSDataInputStream open(Path f, int bufferSize)`
-Implementations MAY throw `UnsupportedOperationException`.
+Implementations without a compliant call SHOULD throw `UnsupportedOperationException`.
#### Preconditions
@@ -944,7 +944,7 @@ to the `DFSFileSystem` implementation is an ongoing matter for debate.
Joins multiple blocks together to create a single file. This
is a little-used operation currently implemented only by HDFS.
-Implementations MAY throw `UnsupportedOperationException`
+Implementations without a compliant call SHOULD throw `UnsupportedOperationException`.
#### Preconditions
@@ -989,7 +989,7 @@ from specification.
Truncate file `p` to the specified `newLength`.
-Implementations MAY throw `UnsupportedOperationException`.
+Implementations without a compliant call SHOULD throw `UnsupportedOperationException`.
#### Preconditions
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 7231349..f8c65da 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -633,7 +633,8 @@ public class S3AFileSystem extends FileSystem {
*/
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by S3AFileSystem");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
index bebb09e..f741298 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
@@ -421,7 +421,8 @@ public class NativeS3FileSystem extends FileSystem {
@Override
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by NativeS3FileSystem");
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: MAPREDUCE-6310. Add jdiff support
to MapReduce. (Li Lu/vinodkv via wangda)
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.6.0.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.6.0.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.6.0.xml
new file mode 100644
index 0000000..166d75f
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.6.0.xml
@@ -0,0 +1,975 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon Jun 13 17:08:52 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="hadoop-mapreduce-client-jobclient 2.6.0"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/hadoop-annotations.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/jdiff.jar -verbose -classpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/classes:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-mapreduce-client-common-2.6.0.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-2.6.0.jar:/Users/vinodkv/Wor
kspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-2.6.0.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-2.6.0.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/target/hadoop-mapreduce-client-shuffle-2.6.0.jar:/Users/vinodkv/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-2.6.0.jar:/Users/vinodkv/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/vinodkv/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/vinodkv/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/vin
odkv/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/vinodkv/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/vinodkv/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/vinodkv/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/vinodkv/.m2/repository/com/google/
inject/guice/3.0/guice-3.0.jar:/Users/vinodkv/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/Users/vinodkv/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/vinodkv/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/vinodkv/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/Users/vinodkv/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/vinodkv/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/vinodkv/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/vinodkv/.m2/repository/jline/jline/0.9.94/jline-0.9.94.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/vinodkv/.m2/repository/jav
ax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/Users/vinodkv/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/vinodkv/.m2/repository/org/htrace/htrace-core/3.0.4/htrace-core-3.0.4.jar:/Users/vinodkv/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/vinodkv/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/vinodkv/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/vinodkv/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-common/target/hadoop-common-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/vinodkv/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/vinodkv/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/vinodkv/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/
jets3t-0.9.0.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/vinodkv/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/vinodkv/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/vinodkv/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/vinodkv/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M
15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-framework/2.6.0/curator-framework-2.6.0.jar:/Users/vinodkv/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-client/2.6.0/curator-client-2.6.0.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-recipes/2.6.0/curator-recipes-2.6.0.jar:/Users/vinodkv/.m2/repository/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-log4j12/1.7.5/slf4j-log4j12-1.7.5.jar:/Users/vinodkv/Workspace/eclipse-work
space/apache-git/hadoop/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.6.0.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home/lib/tools.jar:/Users/vinodkv/.m2/repository/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar:/Users/vinodkv/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/vinodkv/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/vinodkv/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/vinodkv/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/vinodkv/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/vinodkv/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/vinodkv/.m2/repository/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar -sourcepath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-clien
t/hadoop-mapreduce-client-jobclient/src/main/java -apidir /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/site/jdiff/xml -apiname hadoop-mapreduce-client-jobclient 2.6.0 -->
+<package name="org.apache.hadoop.mapred">
+ <!-- start class org.apache.hadoop.mapred.ClientCache -->
+ <class name="ClientCache" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ClientCache" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.ResourceMgrDelegate"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getClient" return="org.apache.hadoop.mapred.ClientServiceDelegate"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+ </method>
+ <method name="getInitializedHSProxy" return="org.apache.hadoop.mapreduce.v2.api.MRClientProtocol"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="instantiateHistoryProxy" return="org.apache.hadoop.mapreduce.v2.api.MRClientProtocol"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ClientCache -->
+ <!-- start class org.apache.hadoop.mapred.ClientServiceDelegate -->
+ <class name="ClientServiceDelegate" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ClientServiceDelegate" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.ResourceMgrDelegate, org.apache.hadoop.mapreduce.JobID, org.apache.hadoop.mapreduce.v2.api.MRClientProtocol"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getMaxClientRetry" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getJobCounters" return="org.apache.hadoop.mapreduce.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="arg1" type="int"/>
+ <param name="arg2" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskDiagnostics" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getJobStatus" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="oldJobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskReports" return="org.apache.hadoop.mapreduce.TaskReport[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="oldJobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="taskType" type="org.apache.hadoop.mapreduce.TaskType"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killTask" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskAttemptID" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <param name="fail" type="boolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killJob" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="oldJobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getLogFilePath" return="org.apache.hadoop.mapreduce.v2.LogParams"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="oldJobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="oldTaskAttemptID" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ClientServiceDelegate -->
+ <!-- start class org.apache.hadoop.mapred.NotRunningJob -->
+ <class name="NotRunningJob" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.v2.api.MRClientProtocol"/>
+ <method name="failTaskAttempt" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getCounters" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getDiagnostics" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getJobReport" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskAttemptCompletionEvents" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskAttemptReport" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskReport" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskReports" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killJob" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killTask" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killTaskAttempt" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getDelegationToken" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="renewDelegationToken" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="cancelDelegationToken" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getConnectAddress" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.NotRunningJob -->
+ <!-- start class org.apache.hadoop.mapred.ResourceMgrDelegate -->
+ <class name="ResourceMgrDelegate" extends="org.apache.hadoop.yarn.client.api.YarnClient"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ResourceMgrDelegate" type="org.apache.hadoop.yarn.conf.YarnConfiguration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Delegate responsible for communicating with the Resource Manager's
+ {@link ApplicationClientProtocol}.
+ @param conf the configuration object.]]>
+ </doc>
+ </constructor>
+ <method name="serviceInit"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <exception name="Exception" type="java.lang.Exception"/>
+ </method>
+ <method name="serviceStart"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <exception name="Exception" type="java.lang.Exception"/>
+ </method>
+ <method name="serviceStop"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <exception name="Exception" type="java.lang.Exception"/>
+ </method>
+ <method name="getActiveTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getAllJobs" return="org.apache.hadoop.mapreduce.JobStatus[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getBlacklistedTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getClusterMetrics" return="org.apache.hadoop.mapreduce.ClusterMetrics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getRMDelegationTokenService" return="org.apache.hadoop.io.Text"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="renewer" type="org.apache.hadoop.io.Text"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getFilesystemName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getNewJobID" return="org.apache.hadoop.mapreduce.JobID"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueue" return="org.apache.hadoop.mapreduce.QueueInfo"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queueName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapreduce.QueueAclsInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getRootQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getChildQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="parent" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getStagingAreaDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getSystemDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskTrackerExpiryInterval" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="setJobPriority"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="arg1" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getProtocolVersion" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="java.lang.String"/>
+ <param name="arg1" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="createApplication" return="org.apache.hadoop.yarn.client.api.YarnClientApplication"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="submitApplication" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appContext" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killApplication"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getAMRMToken" return="org.apache.hadoop.security.token.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplications" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplications" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationTypes" type="java.util.Set"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplications" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationStates" type="java.util.EnumSet"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplications" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationTypes" type="java.util.Set"/>
+ <param name="applicationStates" type="java.util.EnumSet"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getYarnClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getNodeReports" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="states" type="org.apache.hadoop.yarn.api.records.NodeState[]"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="renewer" type="org.apache.hadoop.io.Text"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queueName" type="java.lang.String"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getAllQueues" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getRootQueueInfos" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getChildQueueInfos" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="parent" type="java.lang.String"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getQueueAclsInfo" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplicationAttempts" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getContainers" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="moveApplicationAcrossQueues"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <param name="queue" type="java.lang.String"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getNodeToLabels" return="java.util.Map"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getClusterNodeLabels" return="java.util.Set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <field name="client" type="org.apache.hadoop.yarn.client.api.YarnClient"
+ transient="false" volatile="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ </field>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ResourceMgrDelegate -->
+ <!-- start class org.apache.hadoop.mapred.YarnClientProtocolProvider -->
+ <class name="YarnClientProtocolProvider" extends="org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="YarnClientProtocolProvider"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="create" return="org.apache.hadoop.mapreduce.protocol.ClientProtocol"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="create" return="org.apache.hadoop.mapreduce.protocol.ClientProtocol"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="close"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="clientProtocol" type="org.apache.hadoop.mapreduce.protocol.ClientProtocol"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.YarnClientProtocolProvider -->
+ <!-- start class org.apache.hadoop.mapred.YARNRunner -->
+ <class name="YARNRunner" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.protocol.ClientProtocol"/>
+ <constructor name="YARNRunner" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Yarn runner incapsulates the client interface of
+ yarn
+ @param conf the configuration object for the client]]>
+ </doc>
+ </constructor>
+ <constructor name="YARNRunner" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.ResourceMgrDelegate"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Similar to {@link #YARNRunner(Configuration)} but allowing injecting
+ {@link ResourceMgrDelegate}. Enables mocking and testing.
+ @param conf the configuration object for the client
+ @param resMgrDelegate the resourcemanager client handle.]]>
+ </doc>
+ </constructor>
+ <constructor name="YARNRunner" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.ResourceMgrDelegate, org.apache.hadoop.mapred.ClientCache"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)}
+ but allowing injecting {@link ClientCache}. Enable mocking and testing.
+ @param conf the configuration object
+ @param resMgrDelegate the resource manager delegate
+ @param clientCache the client cache object.]]>
+ </doc>
+ </constructor>
+ <method name="setResourceMgrDelegate"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="resMgrDelegate" type="org.apache.hadoop.mapred.ResourceMgrDelegate"/>
+ </method>
+ <method name="cancelDelegationToken"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.security.token.Token"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getActiveTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getAllJobs" return="org.apache.hadoop.mapreduce.JobStatus[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getBlacklistedTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getClusterMetrics" return="org.apache.hadoop.mapreduce.ClusterMetrics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="renewer" type="org.apache.hadoop.io.Text"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getFilesystemName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getNewJobID" return="org.apache.hadoop.mapreduce.JobID"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueue" return="org.apache.hadoop.mapreduce.QueueInfo"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queueName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapreduce.QueueAclsInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getRootQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getChildQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="parent" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getStagingAreaDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getSystemDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskTrackerExpiryInterval" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="submitJob" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="jobSubmitDir" type="java.lang.String"/>
+ <param name="ts" type="org.apache.hadoop.security.Credentials"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="createApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobConf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="jobSubmitDir" type="java.lang.String"/>
+ <param name="ts" type="org.apache.hadoop.security.Credentials"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setJobPriority"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="arg1" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getProtocolVersion" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="java.lang.String"/>
+ <param name="arg1" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="renewDelegationToken" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.security.token.Token"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getJobCounters" return="org.apache.hadoop.mapreduce.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getJobHistoryDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getJobStatus" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapreduce.TaskCompletionEvent[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="arg1" type="int"/>
+ <param name="arg2" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskDiagnostics" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskReports" return="org.apache.hadoop.mapreduce.TaskReport[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="taskType" type="org.apache.hadoop.mapreduce.TaskType"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="killJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="killTask" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <param name="arg1" type="boolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueueAdmins" return="org.apache.hadoop.security.authorize.AccessControlList"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getProtocolSignature" return="org.apache.hadoop.ipc.ProtocolSignature"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="protocol" type="java.lang.String"/>
+ <param name="clientVersion" type="long"/>
+ <param name="clientMethodsHash" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getLogFileParams" return="org.apache.hadoop.mapreduce.v2.LogParams"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="taskAttemptID" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <doc>
+ <![CDATA[This class enables the current JobClient (0.22 hadoop) to run on YARN.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.YARNRunner -->
+</package>
+
+</api>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: MAPREDUCE-6310. Add jdiff support
to MapReduce. (Li Lu/vinodkv via wangda)
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.7.2.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.7.2.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.7.2.xml
new file mode 100644
index 0000000..dec8dc4
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.7.2.xml
@@ -0,0 +1,990 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Mon Jun 13 20:33:25 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="hadoop-mapreduce-client-jobclient 2.7.2"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/hadoop-annotations.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/jdiff.jar -verbose -classpath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/classes:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/target/hadoop-mapreduce-client-common-2.7.2.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/hadoop-yarn-common-2.7.2.jar:/Users/vinodkv/Wor
kspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/target/hadoop-yarn-client-2.7.2.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/hadoop-mapreduce-client-core-2.7.2.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/target/hadoop-mapreduce-client-shuffle-2.7.2.jar:/Users/vinodkv/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/target/hadoop-yarn-api-2.7.2.jar:/Users/vinodkv/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/vinodkv/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/vinodkv/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/vin
odkv/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/vinodkv/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/vinodkv/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/vinodkv/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/vinodkv/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/vinodkv/.m2/repository/com/google/
inject/guice/3.0/guice-3.0.jar:/Users/vinodkv/.m2/repository/javax/inject/javax.inject/1/javax.inject-1.jar:/Users/vinodkv/.m2/repository/aopalliance/aopalliance/1.0/aopalliance-1.0.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/vinodkv/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/vinodkv/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/vinodkv/.m2/repository/com/sun/jersey/contribs/jersey-guice/1.9/jersey-guice-1.9.jar:/Users/vinodkv/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/vinodkv/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/vinodkv/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/vinodkv/.m2/repository/jline/jline/0.9.94/jline-0.9.94.jar:/Users/vinodkv/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/vinodkv/.m2/repository/xml
enc/xmlenc/0.52/xmlenc-0.52.jar:/Users/vinodkv/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar:/Users/vinodkv/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/vinodkv/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/vinodkv/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/vinodkv/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-common/target/hadoop-common-2.7.2.jar:/Users/vinodkv/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/vinodkv/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/vinodkv/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/vinodkv/.m2/repository/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/Users/vinodkv/.m2/repository/net/
java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/vinodkv/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/vinodkv/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/vinodkv/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/vinodkv/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/vinodkv/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/vinodkv/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.7.2.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/ap
acheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar:/Users/vinodkv/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar:/Users/vinodkv/.m2/repository/org/apache/curator/curator-recipes/2.7.1/curator-recipes-2.7.1.jar:/Users/vinodkv/.m2/repository/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar:/Users/vinodkv/.m2/repository/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar:/Use
rs/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.7.2.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home/lib/tools.jar:/Users/vinodkv/.m2/repository/com/google/inject/extensions/guice-servlet/3.0/guice-servlet-3.0.jar:/Users/vinodkv/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/vinodkv/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/vinodkv/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/vinodkv/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/vinodkv/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/vinodkv/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/vinodkv/.m2/repository/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar -sourcepath /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapredu
ce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java -apidir /Users/vinodkv/Workspace/eclipse-workspace/apache-git/hadoop/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/target/site/jdiff/xml -apiname hadoop-mapreduce-client-jobclient 2.7.2 -->
+<package name="org.apache.hadoop.mapred">
+ <!-- start class org.apache.hadoop.mapred.ClientCache -->
+ <class name="ClientCache" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ClientCache" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.ResourceMgrDelegate"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getClient" return="org.apache.hadoop.mapred.ClientServiceDelegate"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+ </method>
+ <method name="getInitializedHSProxy" return="org.apache.hadoop.mapreduce.v2.api.MRClientProtocol"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="instantiateHistoryProxy" return="org.apache.hadoop.mapreduce.v2.api.MRClientProtocol"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ClientCache -->
+ <!-- start class org.apache.hadoop.mapred.ClientServiceDelegate -->
+ <class name="ClientServiceDelegate" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ClientServiceDelegate" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.ResourceMgrDelegate, org.apache.hadoop.mapreduce.JobID, org.apache.hadoop.mapreduce.v2.api.MRClientProtocol"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getMaxClientRetry" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getJobCounters" return="org.apache.hadoop.mapreduce.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapred.TaskCompletionEvent[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="arg1" type="int"/>
+ <param name="arg2" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskDiagnostics" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getJobStatus" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="oldJobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskReports" return="org.apache.hadoop.mapreduce.TaskReport[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="oldJobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="taskType" type="org.apache.hadoop.mapreduce.TaskType"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killTask" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="taskAttemptID" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <param name="fail" type="boolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killJob" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="oldJobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getLogFilePath" return="org.apache.hadoop.mapreduce.v2.LogParams"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="oldJobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="oldTaskAttemptID" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ClientServiceDelegate -->
+ <!-- start class org.apache.hadoop.mapred.NotRunningJob -->
+ <class name="NotRunningJob" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.v2.api.MRClientProtocol"/>
+ <method name="failTaskAttempt" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getCounters" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getDiagnostics" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getJobReport" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskAttemptCompletionEvents" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskAttemptReport" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptReportRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskReport" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getTaskReports" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killJob" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killTask" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killTaskAttempt" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getDelegationToken" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="renewDelegationToken" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="cancelDelegationToken" return="org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getConnectAddress" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.NotRunningJob -->
+ <!-- start class org.apache.hadoop.mapred.ResourceMgrDelegate -->
+ <class name="ResourceMgrDelegate" extends="org.apache.hadoop.yarn.client.api.YarnClient"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ResourceMgrDelegate" type="org.apache.hadoop.yarn.conf.YarnConfiguration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Delegate responsible for communicating with the Resource Manager's
+ {@link ApplicationClientProtocol}.
+ @param conf the configuration object.]]>
+ </doc>
+ </constructor>
+ <method name="serviceInit"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <exception name="Exception" type="java.lang.Exception"/>
+ </method>
+ <method name="serviceStart"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <exception name="Exception" type="java.lang.Exception"/>
+ </method>
+ <method name="serviceStop"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <exception name="Exception" type="java.lang.Exception"/>
+ </method>
+ <method name="getActiveTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getAllJobs" return="org.apache.hadoop.mapreduce.JobStatus[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getBlacklistedTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getClusterMetrics" return="org.apache.hadoop.mapreduce.ClusterMetrics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getRMDelegationTokenService" return="org.apache.hadoop.io.Text"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="renewer" type="org.apache.hadoop.io.Text"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getFilesystemName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getNewJobID" return="org.apache.hadoop.mapreduce.JobID"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueue" return="org.apache.hadoop.mapreduce.QueueInfo"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queueName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapreduce.QueueAclsInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getRootQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getChildQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="parent" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getStagingAreaDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getSystemDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskTrackerExpiryInterval" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="setJobPriority"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="arg1" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getProtocolVersion" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="java.lang.String"/>
+ <param name="arg1" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplicationId" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="createApplication" return="org.apache.hadoop.yarn.client.api.YarnClientApplication"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="submitApplication" return="org.apache.hadoop.yarn.api.records.ApplicationId"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appContext" type="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="killApplication"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplicationReport" return="org.apache.hadoop.yarn.api.records.ApplicationReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getAMRMToken" return="org.apache.hadoop.security.token.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplications" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplications" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationTypes" type="java.util.Set"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplications" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationStates" type="java.util.EnumSet"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplications" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationTypes" type="java.util.Set"/>
+ <param name="applicationStates" type="java.util.EnumSet"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getYarnClusterMetrics" return="org.apache.hadoop.yarn.api.records.YarnClusterMetrics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getNodeReports" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="states" type="org.apache.hadoop.yarn.api.records.NodeState[]"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getRMDelegationToken" return="org.apache.hadoop.yarn.api.records.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="renewer" type="org.apache.hadoop.io.Text"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getQueueInfo" return="org.apache.hadoop.yarn.api.records.QueueInfo"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queueName" type="java.lang.String"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getAllQueues" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getRootQueueInfos" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getChildQueueInfos" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="parent" type="java.lang.String"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getQueueAclsInfo" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplicationAttemptReport" return="org.apache.hadoop.yarn.api.records.ApplicationAttemptReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getApplicationAttempts" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getContainerReport" return="org.apache.hadoop.yarn.api.records.ContainerReport"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="containerId" type="org.apache.hadoop.yarn.api.records.ContainerId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getContainers" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="applicationAttemptId" type="org.apache.hadoop.yarn.api.records.ApplicationAttemptId"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="moveApplicationAcrossQueues"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="appId" type="org.apache.hadoop.yarn.api.records.ApplicationId"/>
+ <param name="queue" type="java.lang.String"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="submitReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="updateReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="deleteReservation" return="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="request" type="org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getNodeToLabels" return="java.util.Map"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getLabelsToNodes" return="java.util.Map"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getLabelsToNodes" return="java.util.Map"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="labels" type="java.util.Set"/>
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getClusterNodeLabels" return="java.util.Set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="YarnException" type="org.apache.hadoop.yarn.exceptions.YarnException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <field name="client" type="org.apache.hadoop.yarn.client.api.YarnClient"
+ transient="false" volatile="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ </field>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.ResourceMgrDelegate -->
+ <!-- start class org.apache.hadoop.mapred.YarnClientProtocolProvider -->
+ <class name="YarnClientProtocolProvider" extends="org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="YarnClientProtocolProvider"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="create" return="org.apache.hadoop.mapreduce.protocol.ClientProtocol"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="create" return="org.apache.hadoop.mapreduce.protocol.ClientProtocol"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="close"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="clientProtocol" type="org.apache.hadoop.mapreduce.protocol.ClientProtocol"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.YarnClientProtocolProvider -->
+ <!-- start class org.apache.hadoop.mapred.YARNRunner -->
+ <class name="YARNRunner" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.mapreduce.protocol.ClientProtocol"/>
+ <constructor name="YARNRunner" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Yarn runner incapsulates the client interface of
+ yarn
+ @param conf the configuration object for the client]]>
+ </doc>
+ </constructor>
+ <constructor name="YARNRunner" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.ResourceMgrDelegate"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Similar to {@link #YARNRunner(Configuration)} but allowing injecting
+ {@link ResourceMgrDelegate}. Enables mocking and testing.
+ @param conf the configuration object for the client
+ @param resMgrDelegate the resourcemanager client handle.]]>
+ </doc>
+ </constructor>
+ <constructor name="YARNRunner" type="org.apache.hadoop.conf.Configuration, org.apache.hadoop.mapred.ResourceMgrDelegate, org.apache.hadoop.mapred.ClientCache"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)}
+ but allowing injecting {@link ClientCache}. Enable mocking and testing.
+ @param conf the configuration object
+ @param resMgrDelegate the resource manager delegate
+ @param clientCache the client cache object.]]>
+ </doc>
+ </constructor>
+ <method name="setResourceMgrDelegate"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="resMgrDelegate" type="org.apache.hadoop.mapred.ResourceMgrDelegate"/>
+ </method>
+ <method name="cancelDelegationToken"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.security.token.Token"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getActiveTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getAllJobs" return="org.apache.hadoop.mapreduce.JobStatus[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getBlacklistedTrackers" return="org.apache.hadoop.mapreduce.TaskTrackerInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getClusterMetrics" return="org.apache.hadoop.mapreduce.ClusterMetrics"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getDelegationToken" return="org.apache.hadoop.security.token.Token"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="renewer" type="org.apache.hadoop.io.Text"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getFilesystemName" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getNewJobID" return="org.apache.hadoop.mapreduce.JobID"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueue" return="org.apache.hadoop.mapreduce.QueueInfo"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="queueName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueueAclsForCurrentUser" return="org.apache.hadoop.mapreduce.QueueAclsInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getRootQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getChildQueues" return="org.apache.hadoop.mapreduce.QueueInfo[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="parent" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getStagingAreaDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getSystemDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskTrackerExpiryInterval" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="submitJob" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobId" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="jobSubmitDir" type="java.lang.String"/>
+ <param name="ts" type="org.apache.hadoop.security.Credentials"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="createApplicationSubmissionContext" return="org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobConf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="jobSubmitDir" type="java.lang.String"/>
+ <param name="ts" type="org.apache.hadoop.security.Credentials"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="setJobPriority"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="arg1" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getProtocolVersion" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="java.lang.String"/>
+ <param name="arg1" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="renewDelegationToken" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.security.token.Token"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getJobCounters" return="org.apache.hadoop.mapreduce.Counters"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getJobHistoryDir" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getJobStatus" return="org.apache.hadoop.mapreduce.JobStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskCompletionEvents" return="org.apache.hadoop.mapreduce.TaskCompletionEvent[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="arg1" type="int"/>
+ <param name="arg2" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskDiagnostics" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getTaskReports" return="org.apache.hadoop.mapreduce.TaskReport[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="taskType" type="org.apache.hadoop.mapreduce.TaskType"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="killJob"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.JobID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="killTask" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <param name="arg1" type="boolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getQueueAdmins" return="org.apache.hadoop.security.authorize.AccessControlList"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getJobTrackerStatus" return="org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ </method>
+ <method name="getProtocolSignature" return="org.apache.hadoop.ipc.ProtocolSignature"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="protocol" type="java.lang.String"/>
+ <param name="clientVersion" type="long"/>
+ <param name="clientMethodsHash" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getLogFileParams" return="org.apache.hadoop.mapreduce.v2.LogParams"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="jobID" type="org.apache.hadoop.mapreduce.JobID"/>
+ <param name="taskAttemptID" type="org.apache.hadoop.mapreduce.TaskAttemptID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <doc>
+ <![CDATA[This class enables the current JobClient (0.22 hadoop) to run on YARN.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.mapred.YARNRunner -->
+</package>
+
+</api>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/dev-support/jdiff/Null.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Null.java b/hadoop-mapreduce-project/dev-support/jdiff/Null.java
new file mode 100644
index 0000000..7b00145
--- /dev/null
+++ b/hadoop-mapreduce-project/dev-support/jdiff/Null.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class Null {
+ public Null() { }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index 2d969f5..fb3ceb6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -30,6 +30,8 @@
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
<mr.basedir>${project.parent.basedir}/../</mr.basedir>
+ <should.run.jdiff>true</should.run.jdiff>
+ <dev-support.relative.dir>../../dev-support</dev-support.relative.dir>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index b6b356d..4de24c7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -30,6 +30,8 @@
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
<mr.basedir>${project.parent.basedir}/..</mr.basedir>
+ <should.run.jdiff>true</should.run.jdiff>
+ <dev-support.relative.dir>../../dev-support</dev-support.relative.dir>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index bbbe1b2..c4fef7e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -31,6 +31,8 @@
<fork.mode>always</fork.mode>
<!-- Needed for generating FindBugs warnings using parent pom -->
<mr.basedir>${project.parent.basedir}/../</mr.basedir>
+ <should.run.jdiff>true</should.run.jdiff>
+ <dev-support.relative.dir>../../dev-support</dev-support.relative.dir>
</properties>
<dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d937457/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
index 7eabd40..f3fe68b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
@@ -31,6 +31,9 @@
<properties>
<hadoop.common.build.dir>${basedir}/../../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
+ <!-- Used by jdiff -->
+ <!-- Antrun cannot resolve yarn.basedir, so we need to setup something else -->
+ <dev-support.relative.dir>../dev-support</dev-support.relative.dir>
</properties>
<dependencies>
@@ -178,6 +181,130 @@
</plugin>
</plugins>
</build>
+ <profiles>
+ <profile>
+ <id>docs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <jdiff.stable.api>2.7.2</jdiff.stable.api>
+ <jdiff.stability>-unstable</jdiff.stability>
+ <jdiff.javadoc.maxmemory>512m</jdiff.javadoc.maxmemory>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ <phase>prepare-package</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>site</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>jdiff</groupId>
+ <artifactId>jdiff</artifactId>
+ <version>${jdiff.version}</version>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ <destFileName>jdiff.jar</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <version>${project.version}</version>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ <destFileName>hadoop-annotations.jar</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ <version>${xerces.version.jdiff}</version>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ <destFileName>xerces.jar</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>site</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target if="should.run.jdiff">
+
+ <!-- Jdiff -->
+ <mkdir dir="${project.build.directory}/site/jdiff/xml"/>
+ <javadoc maxmemory="${jdiff.javadoc.maxmemory}" verbose="yes">
+ <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
+ path="${project.build.directory}/hadoop-annotations.jar:${project.build.directory}/jdiff.jar">
+ <param name="-apidir" value="${project.build.directory}/site/jdiff/xml"/>
+ <param name="-apiname" value="${project.name} ${project.version}"/>
+ <param name="${jdiff.stability}"/>
+ </doclet>
+ <packageset dir="${basedir}/src/main/java"/>
+ <classpath>
+ <path refid="maven.compile.classpath"/>
+ </classpath>
+ </javadoc>
+ <javadoc sourcepath="${basedir}/src/main/java"
+ destdir="${project.build.directory}/site/jdiff/xml"
+ sourceFiles="${dev-support.relative.dir}/jdiff/Null.java"
+ maxmemory="${jdiff.javadoc.maxmemory}">
+ <doclet name="org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet"
+ path="${project.build.directory}/hadoop-annotations.jar:${project.build.directory}/jdiff.jar:${project.build.directory}/xerces.jar">
+ <param name="-oldapi" value="${project.name} ${jdiff.stable.api}"/>
+ <param name="-newapi" value="${project.name} ${project.version}"/>
+ <param name="-oldapidir" value="${basedir}/${dev-support.relative.dir}/jdiff"/>
+ <param name="-newapidir" value="${project.build.directory}/site/jdiff/xml"/>
+ <param name="-javadocold"
+ value="http://hadoop.apache.org/docs/r${jdiff.stable.api}/api/"/>
+ <param name="-javadocnew" value="${project.build.directory}/site/api/"/>
+ <param name="-stats"/>
+ <param name="${jdiff.stability}"/>
+ </doclet>
+ <classpath>
+ <path refid="maven.compile.classpath"/>
+ </classpath>
+ </javadoc>
+ <property name="compile_classpath" refid="maven.compile.classpath"/>
+
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
<modules>
<module>hadoop-mapreduce-client-core</module>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDFS-10645. Make block report size
as a metric and add this metric to datanode web ui. Contributed by Yuanbo
Liu.
Posted by cd...@apache.org.
HDFS-10645. Make block report size as a metric and add this metric to datanode web ui. Contributed by Yuanbo Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8179f9a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8179f9a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8179f9a4
Branch: refs/heads/HDFS-9806
Commit: 8179f9a493c1b26deb6b1bffacd6a829586b7f98
Parents: dbcaf99
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Aug 19 16:15:00 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Aug 19 16:15:00 2016 +0900
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 1 +
.../hdfs/server/datanode/BPServiceActor.java | 43 +++++++++++++++-
.../hadoop/hdfs/server/datanode/DNConf.java | 7 +++
.../src/main/webapps/datanode/datanode.html | 2 +
.../server/datanode/TestDataNodeMXBean.java | 52 ++++++++++++++++++++
5 files changed, 104 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8179f9a4/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index e4e2443..0666b3f 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -315,6 +315,7 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
| `TotalReadTime` | Total number of milliseconds spent on read operation |
| `RemoteBytesRead` | Number of bytes read by remote clients |
| `RemoteBytesWritten` | Number of bytes written by remote clients |
+| `BPServiceActorInfo` | The information about a block pool service actor |
yarn context
============
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8179f9a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 69989fb..4bde758 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -26,10 +26,13 @@ import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -101,6 +104,9 @@ class BPServiceActor implements Runnable {
private final DataNode dn;
private final DNConf dnConf;
private long prevBlockReportId;
+ private final SortedSet<Integer> blockReportSizes =
+ Collections.synchronizedSortedSet(new TreeSet<>());
+ private final int maxDataLength;
private final IncrementalBlockReportManager ibrManager;
@@ -122,6 +128,8 @@ class BPServiceActor implements Runnable {
prevBlockReportId = ThreadLocalRandom.current().nextLong();
scheduler = new Scheduler(dnConf.heartBeatInterval,
dnConf.getLifelineIntervalMs(), dnConf.blockReportInterval);
+ // get the value of maxDataLength.
+ this.maxDataLength = dnConf.getMaxDataLength();
}
public DatanodeRegistration getBpRegistration() {
@@ -166,6 +174,8 @@ class BPServiceActor implements Runnable {
String.valueOf(getScheduler().getLastHearbeatTime()));
info.put("LastBlockReport",
String.valueOf(getScheduler().getLastBlockReportTime()));
+ info.put("maxBlockReportSize", String.valueOf(getMaxBlockReportSize()));
+ info.put("maxDataLength", String.valueOf(maxDataLength));
return info;
}
@@ -305,6 +315,14 @@ class BPServiceActor implements Runnable {
}
}
+ private int getMaxBlockReportSize() {
+ int maxBlockReportSize = 0;
+ if (!blockReportSizes.isEmpty()) {
+ maxBlockReportSize = blockReportSizes.last();
+ }
+ return maxBlockReportSize;
+ }
+
private long generateUniqueBlockReportId() {
// Initialize the block report ID the first time through.
// Note that 0 is used on the NN to indicate "uninitialized", so we should
@@ -353,12 +371,18 @@ class BPServiceActor implements Runnable {
boolean success = false;
long brSendStartTime = monotonicNow();
long reportId = generateUniqueBlockReportId();
+ boolean useBlocksBuffer =
+ bpRegistration.getNamespaceInfo().isCapabilitySupported(
+ NamespaceInfo.Capability.STORAGE_BLOCK_REPORT_BUFFERS);
+ blockReportSizes.clear();
try {
if (totalBlockCount < dnConf.blockReportSplitThreshold) {
// Below split threshold, send all reports in a single message.
DatanodeCommand cmd = bpNamenode.blockReport(
bpRegistration, bpos.getBlockPoolId(), reports,
new BlockReportContext(1, 0, reportId, fullBrLeaseId, true));
+ blockReportSizes.add(
+ calculateBlockReportPBSize(useBlocksBuffer, reports));
numRPCs = 1;
numReportsSent = reports.length;
if (cmd != null) {
@@ -372,6 +396,8 @@ class BPServiceActor implements Runnable {
bpRegistration, bpos.getBlockPoolId(), singleReport,
new BlockReportContext(reports.length, r, reportId,
fullBrLeaseId, true));
+ blockReportSizes.add(
+ calculateBlockReportPBSize(useBlocksBuffer, singleReport));
numReportsSent++;
numRPCs++;
if (cmd != null) {
@@ -437,7 +463,22 @@ class BPServiceActor implements Runnable {
}
return cmd;
}
-
+
+ private int calculateBlockReportPBSize(
+ boolean useBlocksBuffer, StorageBlockReport[] reports) {
+ int reportSize = 0;
+
+ for (StorageBlockReport r : reports) {
+ if (useBlocksBuffer) {
+ reportSize += r.getBlocks().getBlocksBuffer().size();
+ } else {
+ // each block costs 10 bytes in PB because of uint64
+ reportSize += 10 * r.getBlocks().getBlockListAsLongs().length;
+ }
+ }
+ return reportSize;
+ }
+
HeartbeatResponse sendHeartBeat(boolean requestBlockReportLease)
throws IOException {
scheduler.scheduleNextHeartbeat();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8179f9a4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 942672e..11197e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -117,6 +117,7 @@ public class DNConf {
private final int volFailuresTolerated;
private final int volsConfigured;
+ private final int maxDataLength;
public DNConf(Configuration conf) {
this.conf = conf;
@@ -149,6 +150,8 @@ public class DNConf {
readaheadLength = conf.getLong(
HdfsClientConfigKeys.DFS_DATANODE_READAHEAD_BYTES_KEY,
HdfsClientConfigKeys.DFS_DATANODE_READAHEAD_BYTES_DEFAULT);
+ maxDataLength = conf.getInt(DFSConfigKeys.IPC_MAXIMUM_DATA_LENGTH,
+ DFSConfigKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT);
dropCacheBehindWrites = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_KEY,
DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_DEFAULT);
@@ -389,4 +392,8 @@ public class DNConf {
public int getVolsConfigured() {
return volsConfigured;
}
+
+ int getMaxDataLength() {
+ return maxDataLength;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8179f9a4/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
index 22a2733..b35a0a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
@@ -80,6 +80,7 @@
<th>Actor State</th>
<th>Last Heartbeat</th>
<th>Last Block Report</th>
+ <th>Last Block Report Size (Max Size)</th>
</tr>
</thead>
{#dn.BPServiceActorInfo}
@@ -89,6 +90,7 @@
<td>{ActorState}</td>
<td>{LastHeartbeat}s</td>
<td>{#helper_relative_time value="{LastBlockReport}"/}</td>
+ <td>{maxBlockReportSize|fmt_bytes} ({maxDataLength|fmt_bytes})</td>
</tr>
{/dn.BPServiceActorInfo}
</table>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8179f9a4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
index e7b2c7c..8b0d5cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.datanode;
import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -26,22 +27,31 @@ import javax.management.MBeanServer;
import javax.management.ObjectName;
import com.google.common.base.Supplier;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.test.GenericTestUtils;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
import org.junit.Assert;
import org.junit.Test;
import org.mortbay.util.ajax.JSON;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Class for testing {@link DataNodeMXBean} implementation
*/
public class TestDataNodeMXBean {
+
+ public static final Log LOG = LogFactory.getLog(TestDataNodeMXBean.class);
+
@Test
public void testDataNodeMXBean() throws Exception {
Configuration conf = new Configuration();
@@ -100,6 +110,48 @@ public class TestDataNodeMXBean {
}
@Test
+ public void testDataNodeMXBeanBlockSize() throws Exception {
+ Configuration conf = new Configuration();
+
+ try(MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).build()) {
+ DataNode dn = cluster.getDataNodes().get(0);
+ for (int i = 0; i < 100; i++) {
+ DFSTestUtil.writeFile(
+ cluster.getFileSystem(),
+ new Path("/foo" + String.valueOf(i) + ".txt"), "test content");
+ }
+ DataNodeTestUtils.triggerBlockReport(dn);
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanName = new ObjectName(
+ "Hadoop:service=DataNode,name=DataNodeInfo");
+ String bpActorInfo = (String)mbs.getAttribute(mxbeanName,
+ "BPServiceActorInfo");
+ Assert.assertEquals(dn.getBPServiceActorInfo(), bpActorInfo);
+ LOG.info("bpActorInfo is " + bpActorInfo);
+ TypeReference<ArrayList<Map<String, String>>> typeRef
+ = new TypeReference<ArrayList<Map<String, String>>>() {};
+ ArrayList<Map<String, String>> bpActorInfoList =
+ new ObjectMapper().readValue(bpActorInfo, typeRef);
+ int maxDataLength =
+ Integer.valueOf(bpActorInfoList.get(0).get("maxDataLength"));
+ int confMaxDataLength = dn.getConf().getInt(
+ CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH,
+ CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT);
+ int maxBlockReportSize =
+ Integer.valueOf(bpActorInfoList.get(0).get("maxBlockReportSize"));
+ LOG.info("maxDataLength is " + maxDataLength);
+ LOG.info("maxBlockReportSize is " + maxBlockReportSize);
+ assertTrue("maxBlockReportSize should be greater than zero",
+ maxBlockReportSize > 0);
+ assertEquals("maxDataLength should be exactly "
+ + "the same value of ipc.maximum.data.length",
+ confMaxDataLength,
+ maxDataLength);
+ }
+ }
+
+ @Test
public void testDataNodeMXBeanBlockCount() throws Exception {
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HADOOP-13428. Fixed hadoop-common
build files to generate jdiff by working around a jdiff-bug. Contributed by
Wangda Tan.
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99603e90/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.7.2.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.7.2.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.7.2.xml
new file mode 100644
index 0000000..5ef99b2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.7.2.xml
@@ -0,0 +1,46648 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Thu Aug 18 16:00:16 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="Apache Hadoop Common 2.7.2"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/jdiff.jar -verbose -classpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/classes:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.7.2.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/lib/tools.jar:/Users/wtan/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/wtan/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/wtan/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/wtan/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/wtan/.m2/repository/commons-
httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/wtan/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/wtan/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/wtan/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/wtan/.m2/repository/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/Users/wtan/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/wtan/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/wtan/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/wtan/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/wtan/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/wtan/.m2/repository/javax/xml/bind/jaxb-ap
i/2.2.2/jaxb-api-2.2.2.jar:/Users/wtan/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/wtan/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/wtan/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/wtan/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/wtan/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/wtan/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/wtan/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/wtan/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/wtan/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.ja
r:/Users/wtan/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/wtan/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/wtan/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/wtan/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/wtan/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/wtan/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/wtan/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/wtan/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/wtan/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1
.jar:/Users/wtan/.m2/repository/org/apache/ant/ant/1.8.1/ant-1.8.1.jar:/Users/wtan/.m2/repository/org/apache/ant/ant-launcher/1.8.1/ant-launcher-1.8.1.jar:/Users/wtan/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/wtan/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.7.2.jar:/Users/wtan/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/wtan/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/wtan/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/wtan/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar:/Users/wtan/.m2/repository/com/jcraft/j
sch/0.1.42/jsch-0.1.42.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-recipes/2.7.1/curator-recipes-2.7.1.jar:/Users/wtan/.m2/repository/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar:/Users/wtan/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar:/Users/wtan/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/wtan/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/wtan/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/wtan/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar -sourcepath /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/src/main/java -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-c
ommon/target/hadoop-annotations.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/jdiff.jar -apidir /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/site/jdiff/xml -apiname Apache Hadoop Common 2.7.2 -->
+<package name="org.apache.hadoop">
+ <!-- start class org.apache.hadoop.HadoopIllegalArgumentException -->
+ <class name="HadoopIllegalArgumentException" extends="java.lang.IllegalArgumentException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HadoopIllegalArgumentException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Constructs exception with the specified detail message.
+ @param message detailed message.]]>
+ </doc>
+ </constructor>
+ <doc>
+ <![CDATA[Indicates that a method has been passed illegal or invalid argument. This
+ exception is thrown instead of IllegalArgumentException to differentiate the
+ exception thrown in Hadoop implementation from the one thrown in JDK.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.HadoopIllegalArgumentException -->
+</package>
+<package name="org.apache.hadoop.conf">
+ <!-- start interface org.apache.hadoop.conf.Configurable -->
+ <interface name="Configurable" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="setConf"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Set the configuration to be used by this object.]]>
+ </doc>
+ </method>
+ <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return the configuration used by this object.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Something that may be configured with a {@link Configuration}.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.conf.Configurable -->
+ <!-- start class org.apache.hadoop.conf.Configuration -->
+ <class name="Configuration" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="java.lang.Iterable"/>
+ <implements name="org.apache.hadoop.io.Writable"/>
+ <constructor name="Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[A new configuration.]]>
+ </doc>
+ </constructor>
+ <constructor name="Configuration" type="boolean"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[A new configuration where the behavior of reading from the default
+ resources can be turned off.
+
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files.
+ @param loadDefaults specifies whether to load from the default files]]>
+ </doc>
+ </constructor>
+ <constructor name="Configuration" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[A new configuration with the same settings cloned from another.
+
+ @param other the configuration from which to clone settings.]]>
+ </doc>
+ </constructor>
+ <method name="addDeprecations"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="deltas" type="org.apache.hadoop.conf.Configuration.DeprecationDelta[]"/>
+ <doc>
+ <![CDATA[Adds a set of deprecated keys to the global deprecations.
+
+ This method is lockless. It works by means of creating a new
+ DeprecationContext based on the old one, and then atomically swapping in
+ the new context. If someone else updated the context in between us reading
+ the old context and swapping in the new one, we try again until we win the
+ race.
+
+ @param deltas The deprecations to add.]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="use {@link #addDeprecation(String key, String newKey,
+ String customMessage)} instead">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKeys" type="java.lang.String[]"/>
+ <param name="customMessage" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If a key is deprecated in favor of multiple keys, they are all treated as
+ aliases of each other, and setting any one of them resets all the others
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key
+ @param newKeys
+ @param customMessage
+ @deprecated use {@link #addDeprecation(String key, String newKey,
+ String customMessage)} instead]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKey" type="java.lang.String"/>
+ <param name="customMessage" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key
+ @param newKey
+ @param customMessage]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="use {@link #addDeprecation(String key, String newKey)} instead">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKeys" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If a key is deprecated in favor of multiple keys, they are all treated as
+ aliases of each other, and setting any one of them resets all the others
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKeys list of keys that take up the values of deprecated key
+ @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKey" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKey key that takes up the value of deprecated key]]>
+ </doc>
+ </method>
+ <method name="isDeprecated" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.String"/>
+ <doc>
+ <![CDATA[checks whether the given <code>key</code> is deprecated.
+
+ @param key the parameter which is to be checked for deprecation
+ @return <code>true</code> if the key is deprecated and
+ <code>false</code> otherwise.]]>
+ </doc>
+ </method>
+ <method name="setDeprecatedProperties"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Sets all deprecated properties that are not currently set but have a
+ corresponding new property that is set. Useful for iterating the
+ properties when all deprecated properties for currently set properties
+ need to be present.]]>
+ </doc>
+ </method>
+ <method name="addDefaultResource"
+ abstract="false" native="false" synchronized="true"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a default resource. Resources are loaded in the order of the resources
+ added.
+ @param name file name. File should be present in the classpath.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param name resource to be added, the classpath is examined for a file
+ with that name.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="url" type="java.net.URL"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param url url of the resource to be added, the local filesystem is
+ examined directly to find the resource, without referring to
+ the classpath.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="file" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param file file-path of resource to be added, the local filesystem is
+ examined directly to find the resource, without referring to
+ the classpath.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.InputStream"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ WARNING: The contents of the InputStream will be cached, by this method.
+ So use this sparingly because it does increase the memory consumption.
+
+ @param in InputStream to deserialize the object from. In will be read from
+ when a get or set is called next. After it is read the stream will be
+ closed.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.InputStream"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param in InputStream to deserialize the object from.
+ @param name the name of the resource because InputStream.toString is not
+ very descriptive some times.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param conf Configuration object from which to load properties]]>
+ </doc>
+ </method>
+ <method name="reloadConfiguration"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Reload configuration from previously added resources.
+
+ This method will clear all the configuration read from the added
+ resources, and final parameters. This will make the resources to
+ be read again before accessing the values. Values that are added
+ via set methods will overlay values read from the resources.]]>
+ </doc>
+ </method>
+ <method name="get" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property, <code>null</code> if
+ no such property exists. If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null.
+
+ Values are processed for <a href="#VariableExpansion">variable expansion</a>
+ before being returned.
+
+ @param name the property name, will be trimmed before get value.
+ @return the value of the <code>name</code> or its replacing property,
+ or null if no such property exists.]]>
+ </doc>
+ </method>
+ <method name="setAllowNullValueProperties"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="val" type="boolean"/>
+ <doc>
+ <![CDATA[Set Configuration to allow keys without values during setup. Intended
+ for use during testing.
+
+ @param val If true, will allow Configuration to store keys without values]]>
+ </doc>
+ </method>
+ <method name="onlyKeyExists" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Return existence of the <code>name</code> property, but only for
+ names which have no valid value, usually non-existent or commented
+ out in XML.
+
+ @param name the property name
+ @return true if the property <code>name</code> exists without value]]>
+ </doc>
+ </method>
+ <method name="getTrimmed" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>,
+ <code>null</code> if no such property exists.
+ If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null
+
+ Values are processed for <a href="#VariableExpansion">variable expansion</a>
+ before being returned.
+
+ @param name the property name.
+ @return the value of the <code>name</code> or its replacing property,
+ or null if no such property exists.]]>
+ </doc>
+ </method>
+ <method name="getTrimmed" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>,
+ <code>defaultValue</code> if no such property exists.
+ See @{Configuration#getTrimmed} for more details.
+
+ @param name the property name.
+ @param defaultValue the property default value.
+ @return the value of the <code>name</code> or defaultValue
+ if it is not set.]]>
+ </doc>
+ </method>
+ <method name="getRaw" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property, without doing
+ <a href="#VariableExpansion">variable expansion</a>.If the key is
+ deprecated, it returns the value of the first key which replaces
+ the deprecated key and is not null.
+
+ @param name the property name.
+ @return the value of the <code>name</code> property or
+ its replacing property and null if no such property exists.]]>
+ </doc>
+ </method>
+ <method name="set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the <code>value</code> of the <code>name</code> property. If
+ <code>name</code> is deprecated or there is a deprecated name associated to it,
+ it sets the value to both names. Name will be trimmed before put into
+ configuration.
+
+ @param name property name.
+ @param value property value.]]>
+ </doc>
+ </method>
+ <method name="set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="java.lang.String"/>
+ <param name="source" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the <code>value</code> of the <code>name</code> property. If
+ <code>name</code> is deprecated, it also sets the <code>value</code> to
+ the keys that replace the deprecated key. Name will be trimmed before put
+ into configuration.
+
+ @param name property name.
+ @param value property value.
+ @param source the place that this configuration value came from
+ (For debugging).
+ @throws IllegalArgumentException when the value or name is null.]]>
+ </doc>
+ </method>
+ <method name="unset"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Unset a previously set property.]]>
+ </doc>
+ </method>
+ <method name="setIfUnset"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Sets a property if it is currently unset.
+ @param name the property name
+ @param value the new value]]>
+ </doc>
+ </method>
+ <method name="get" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code>. If the key is deprecated,
+ it returns the value of the first key which replaces the deprecated key
+ and is not null.
+ If no such property exists,
+ then <code>defaultValue</code> is returned.
+
+ @param name property name, will be trimmed before get value.
+ @param defaultValue default value.
+ @return property value, or <code>defaultValue</code> if the property
+ doesn't exist.]]>
+ </doc>
+ </method>
+ <method name="getInt" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="int"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as an <code>int</code>.
+
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>int</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as an <code>int</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getInts" return="int[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a set of comma-delimited
+ <code>int</code> values.
+
+ If no such property exists, an empty array is returned.
+
+ @param name property name
+ @return property value interpreted as an array of comma-delimited
+ <code>int</code> values]]>
+ </doc>
+ </method>
+ <method name="setInt"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="int"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to an <code>int</code>.
+
+ @param name property name.
+ @param value <code>int</code> value of the property.]]>
+ </doc>
+ </method>
+ <method name="getLong" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="long"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>long</code>.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>long</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getLongBytes" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="long"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>long</code> or
+ human readable format. If no such property exists, the provided default
+ value is returned, or if the specified value is not a valid
+ <code>long</code> or human readable format, then an error is thrown. You
+ can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ t(tera), p(peta), e(exa)
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setLong"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="long"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>long</code>.
+
+ @param name property name.
+ @param value <code>long</code> value of the property.]]>
+ </doc>
+ </method>
+ <method name="getFloat" return="float"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="float"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>float</code>.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>float</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>float</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setFloat"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="float"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>float</code>.
+
+ @param name property name.
+ @param value property value.]]>
+ </doc>
+ </method>
+ <method name="getDouble" return="double"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="double"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>double</code>.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>double</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>double</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setDouble"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="double"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>double</code>.
+
+ @param name property name.
+ @param value property value.]]>
+ </doc>
+ </method>
+ <method name="getBoolean" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="boolean"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>boolean</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>boolean</code>, then <code>defaultValue</code> is returned.
+
+ @param name property name.
+ @param defaultValue default value.
+ @return property value as a <code>boolean</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setBoolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="boolean"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>boolean</code>.
+
+ @param name property name.
+ @param value <code>boolean</code> value of the property.]]>
+ </doc>
+ </method>
+ <method name="setBooleanIfUnset"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="boolean"/>
+ <doc>
+ <![CDATA[Set the given property, if it is currently unset.
+ @param name property name
+ @param value new value]]>
+ </doc>
+ </method>
+ <method name="setEnum"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="T"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to the given type. This
+ is equivalent to <code>set(<name>, value.toString())</code>.
+ @param name property name
+ @param value new value]]>
+ </doc>
+ </method>
+ <method name="getEnum" return="T"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="T"/>
+ <doc>
+ <![CDATA[Return value matching this enumerated type.
+ Note that the returned value is trimmed by this method.
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists
+ @throws IllegalArgumentException If mapping is illegal for the type
+ provided]]>
+ </doc>
+ </method>
+ <method name="setTimeDuration"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="long"/>
+ <param name="unit" type="java.util.concurrent.TimeUnit"/>
+ <doc>
+ <![CDATA[Set the value of <code>name</code> to the given time duration. This
+ is equivalent to <code>set(<name>, value + <time suffix>)</code>.
+ @param name Property name
+ @param value Time duration
+ @param unit Unit of time]]>
+ </doc>
+ </method>
+ <method name="getTimeDuration" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="long"/>
+ <param name="unit" type="java.util.concurrent.TimeUnit"/>
+ <doc>
+ <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param unit Unit to convert the stored property, if it exists.
+ @throws NumberFormatException If the property stripped of its unit is not
+ a number]]>
+ </doc>
+ </method>
+ <method name="getPattern" return="java.util.regex.Pattern"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.util.regex.Pattern"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>Pattern</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>Pattern</code>, then <code>DefaultValue</code> is returned.
+ Note that the returned value is NOT trimmed by this method.
+
+ @param name property name
+ @param defaultValue default value
+ @return property value as a compiled Pattern, or defaultValue]]>
+ </doc>
+ </method>
+ <method name="setPattern"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="pattern" type="java.util.regex.Pattern"/>
+ <doc>
+ <![CDATA[Set the given property to <code>Pattern</code>.
+ If the pattern is passed as null, sets the empty pattern which results in
+ further calls to getPattern(...) returning the default value.
+
+ @param name property name
+ @param pattern new value]]>
+ </doc>
+ </method>
+ <method name="getPropertySources" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Gets information about why a property was set. Typically this is the
+ path to the resource objects (file, URL, etc.) the property came from, but
+ it can also indicate that it was set programatically, or because of the
+ command line.
+
+ @param name - The property name to get the source of.
+ @return null - If the property or its source wasn't found. Otherwise,
+ returns a list of the sources of the resource. The older sources are
+ the first ones in the list. So for example if a configuration is set from
+ the command line, and then written out to a file that is read back in the
+ first entry would indicate that it was set from the command line, while
+ the second one would indicate the file that the new configuration was read
+ in from.]]>
+ </doc>
+ </method>
+ <method name="getRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Parse the given attribute as a set of integer ranges
+ @param name the attribute name
+ @param defaultValue the default value if it is not set
+ @return a new set of ranges from the configured value]]>
+ </doc>
+ </method>
+ <method name="getStringCollection" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ a collection of <code>String</code>s.
+ If no such property is specified then empty collection is returned.
+ <p>
+ This is an optimized version of {@link #getStrings(String)}
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s.]]>
+ </doc>
+ </method>
+ <method name="getStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s.
+ If no such property is specified then <code>null</code> is returned.
+
+ @param name property name.
+ @return property value as an array of <code>String</code>s,
+ or <code>null</code>.]]>
+ </doc>
+ </method>
+ <method name="getStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s.
+ If no such property is specified then default value is returned.
+
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of <code>String</code>s,
+ or default value.]]>
+ </doc>
+ </method>
+ <method name="getTrimmedStringCollection" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then empty <code>Collection</code> is returned.
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>]]>
+ </doc>
+ </method>
+ <method name="getTrimmedStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then an empty array is returned.
+
+ @param name property name.
+ @return property value as an array of trimmed <code>String</code>s,
+ or empty array.]]>
+ </doc>
+ </method>
+ <method name="getTrimmedStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then default value is returned.
+
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of trimmed <code>String</code>s,
+ or default value.]]>
+ </doc>
+ </method>
+ <method name="setStrings"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="values" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Set the array of string values for the <code>name</code> property as
+ as comma delimited values.
+
+ @param name property name.
+ @param values The values]]>
+ </doc>
+ </method>
+ <method name="getPassword" return="char[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the value for a known password configuration element.
+ In order to enable the elimination of clear text passwords in config,
+ this method attempts to resolve the property name as an alias through
+ the CredentialProvider API and conditionally fallsback to config.
+ @param name property name
+ @return password]]>
+ </doc>
+ </method>
+ <method name="getPasswordFromCredentialProviders" return="char[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Try and resolve the provided element name as a credential provider
+ alias.
+ @param name alias of the provisioned credential
+ @return password or null if not found
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getPasswordFromConfig" return="char[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Fallback to clear text passwords in configuration.
+ @param name
+ @return clear text password or null]]>
+ </doc>
+ </method>
+ <method name="getSocketAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="hostProperty" type="java.lang.String"/>
+ <param name="addressProperty" type="java.lang.String"/>
+ <param name="defaultAddressValue" type="java.lang.String"/>
+ <param name="defaultPort" type="int"/>
+ <doc>
+ <![CDATA[Get the socket address for <code>hostProperty</code> as a
+ <code>InetSocketAddress</code>. If <code>hostProperty</code> is
+ <code>null</code>, <code>addressProperty</code> will be used. This
+ is useful for cases where we want to differentiate between host
+ bind address and address clients should use to establish connection.
+
+ @param hostProperty bind host property name.
+ @param addressProperty address property name.
+ @param defaultAddressValue the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+ </doc>
+ </method>
+ <method name="getSocketAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultAddress" type="java.lang.String"/>
+ <param name="defaultPort" type="int"/>
+ <doc>
+ <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+ </doc>
+ </method>
+ <method name="setSocketAddr"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <doc>
+ <![CDATA[Set the socket address for the <code>name</code> property as
+ a <code>host:port</code>.]]>
+ </doc>
+ </method>
+ <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="hostProperty" type="java.lang.String"/>
+ <param name="addressProperty" type="java.lang.String"/>
+ <param name="defaultAddressValue" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <doc>
+ <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>. The wildcard
+ address is replaced with the local host's address. If the host and address
+ properties are configured the host component of the address will be combined
+ with the port component of the addr to generate the address. This is to allow
+ optional control over which host name is used in multi-home bind-host
+ cases where a host can have multiple names
+ @param hostProperty the bind-host configuration name
+ @param addressProperty the service address configuration name
+ @param defaultAddressValue the service default address configuration value
+ @param addr InetSocketAddress of the service listener
+ @return InetSocketAddress for clients to connect]]>
+ </doc>
+ </method>
+ <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <doc>
+ <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>. The wildcard
+ address is replaced with the local host's address.
+ @param name property name.
+ @param addr InetSocketAddress of a listener to store in the given property
+ @return InetSocketAddress for clients to connect]]>
+ </doc>
+ </method>
+ <method name="getClassByName" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+ <doc>
+ <![CDATA[Load a class by name.
+
+ @param name the class name.
+ @return the class object.
+ @throws ClassNotFoundException if the class is not found.]]>
+ </doc>
+ </method>
+ <method name="getClassByNameOrNull" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Load a class by name, returning null rather than throwing an exception
+ if it couldn't be loaded. This is to avoid the overhead of creating
+ an exception.
+
+ @param name the class name
+ @return the class object, or null if it could not be found.]]>
+ </doc>
+ </method>
+ <method name="getClasses" return="java.lang.Class[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.Class[]"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property
+ as an array of <code>Class</code>.
+ The value of the property specifies a list of comma separated class names.
+ If no such property is specified, then <code>defaultValue</code> is
+ returned.
+
+ @param name the property name.
+ @param defaultValue default value.
+ @return property value as a <code>Class[]</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>.
+ If no such property is specified, then <code>defaultValue</code> is
+ returned.
+
+ @param name the class name.
+ @param defaultValue default value.
+ @return property value as a <code>Class</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.Class"/>
+ <param name="xface" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>
+ implementing the interface specified by <code>xface</code>.
+
+ If no such property is specified, then <code>defaultValue</code> is
+ returned.
+
+ An exception is thrown if the returned class does not implement the named
+ interface.
+
+ @param name the class name.
+ @param defaultValue default value.
+ @param xface the interface implemented by the named class.
+ @return property value as a <code>Class</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getInstances" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="xface" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>List</code>
+ of objects implementing the interface specified by <code>xface</code>.
+
+ An exception is thrown if any of the classes does not exist, or if it does
+ not implement the named interface.
+
+ @param name the property name.
+ @param xface the interface implemented by the classes named by
+ <code>name</code>.
+ @return a <code>List</code> of objects implementing <code>xface</code>.]]>
+ </doc>
+ </method>
+ <method name="setClass"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="theClass" type="java.lang.Class"/>
+ <param name="xface" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to the name of a
+ <code>theClass</code> implementing the given interface <code>xface</code>.
+
+ An exception is thrown if <code>theClass</code> does not implement the
+ interface <code>xface</code>.
+
+ @param name property name.
+ @param theClass property value.
+ @param xface the interface implemented by the named class.]]>
+ </doc>
+ </method>
+ <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="dirsProp" type="java.lang.String"/>
+ <param name="path" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get a local file under a directory named by <i>dirsProp</i> with
+ the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code. If the selected
+ directory does not exist, an attempt is made to create it.
+
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+ </doc>
+ </method>
+ <method name="getFile" return="java.io.File"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="dirsProp" type="java.lang.String"/>
+ <param name="path" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get a local file name under a directory named in <i>dirsProp</i> with
+ the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code. If the selected
+ directory does not exist, an attempt is made to create it.
+
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+ </doc>
+ </method>
+ <method name="getResource" return="java.net.URL"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the {@link URL} for the named resource.
+
+ @param name resource name.
+ @return the url for the named resource.]]>
+ </doc>
+ </method>
+ <method name="getConfResourceAsInputStream" return="java.io.InputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get an input stream attached to the configuration resource with the
+ given <code>name</code>.
+
+ @param name configuration resource name.
+ @return an input stream attached to the resource.]]>
+ </doc>
+ </method>
+ <method name="getConfResourceAsReader" return="java.io.Reader"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get a {@link Reader} attached to the configuration resource with the
+ given <code>name</code>.
+
+ @param name configuration resource name.
+ @return a reader attached to the resource.]]>
+ </doc>
+ </method>
+ <method name="getFinalParameters" return="java.util.Set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the set of parameters marked final.
+
+ @return final parameter set.]]>
+ </doc>
+ </method>
+ <method name="getProps" return="java.util.Properties"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ </method>
+ <method name="size" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return the number of keys in the configuration.
+
+ @return number of keys in the configuration.]]>
+ </doc>
+ </method>
+ <method name="clear"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Clears all keys from the configuration.]]>
+ </doc>
+ </method>
+ <method name="iterator" return="java.util.Iterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get an {@link Iterator} to go through the list of <code>String</code>
+ key-value pairs in the configuration.
+
+ @return an iterator over the entries.]]>
+ </doc>
+ </method>
+ <method name="writeXml"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.OutputStream"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link OutputStream} using UTF-8 encoding.
+
+ @param out the output stream to write to.]]>
+ </doc>
+ </method>
+ <method name="writeXml"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.Writer"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link Writer}.
+
+ @param out the writer to write to.]]>
+ </doc>
+ </method>
+ <method name="dumpConfiguration"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="out" type="java.io.Writer"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Writes out all the parameters and their properties (final and resource) to
+ the given {@link Writer}
+ The format of the output would be
+ { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
+ key2.isFinal,key2.resource}... ] }
+ It does not output the parameters of the configuration object which is
+ loaded from an input stream.
+ @param out the Writer to write to
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getClassLoader" return="java.lang.ClassLoader"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the {@link ClassLoader} for this job.
+
+ @return the correct class loader.]]>
+ </doc>
+ </method>
+ <method name="setClassLoader"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="classLoader" type="java.lang.ClassLoader"/>
+ <doc>
+ <![CDATA[Set the class loader that will be used to load the various objects.
+
+ @param classLoader the new class loader.]]>
+ </doc>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setQuietMode"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="quietmode" type="boolean"/>
+ <doc>
+ <![CDATA[Set the quietness-mode.
+
+ In the quiet-mode, error and informational messages might not be logged.
+
+ @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
+ to turn it off.]]>
+ </doc>
+ </method>
+ <method name="main"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="args" type="java.lang.String[]"/>
+ <exception name="Exception" type="java.lang.Exception"/>
+ <doc>
+ <![CDATA[For debugging. List non-default properties to the terminal and exit.]]>
+ </doc>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getValByRegex" return="java.util.Map"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="regex" type="java.lang.String"/>
+ <doc>
+ <![CDATA[get keys matching the the regex
+ @param regex
+ @return Map<String,String> with matching keys]]>
+ </doc>
+ </method>
+ <method name="dumpDeprecatedKeys"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="hasWarnedDeprecation" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Returns whether or not a deprecated name has been warned. If the name is not
+ deprecated then always return false]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Provides access to configuration parameters.
+
+ <h4 id="Resources">Resources</h4>
+
+ <p>Configurations are specified by resources. A resource contains a set of
+ name/value pairs as XML data. Each resource is named by either a
+ <code>String</code> or by a {@link Path}. If named by a <code>String</code>,
+ then the classpath is examined for a file with that name. If named by a
+ <code>Path</code>, then the local filesystem is examined directly, without
+ referring to the classpath.
+
+ <p>Unless explicitly turned off, Hadoop by default specifies two
+ resources, loaded in-order from the classpath: <ol>
+ <li><tt>
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
+ <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
+ installation.</li>
+ </ol>
+ Applications may add additional resources, which are loaded
+ subsequent to these resources in the order they are added.
+
+ <h4 id="FinalParams">Final Parameters</h4>
+
+ <p>Configuration parameters may be declared <i>final</i>.
+ Once a resource declares a value final, no subsequently-loaded
+ resource can alter that value.
+ For example, one might define a final parameter with:
+ <tt><pre>
+ <property>
+ <name>dfs.hosts.include</name>
+ <value>/etc/hadoop/conf/hosts.include</value>
+ <b><final>true</final></b>
+ </property></pre></tt>
+
+ Administrators typically define parameters as final in
+ <tt>core-site.xml</tt> for values that user applications may not alter.
+
+ <h4 id="VariableExpansion">Variable Expansion</h4>
+
+ <p>Value strings are first processed for <i>variable expansion</i>. The
+ available properties are:<ol>
+ <li>Other properties defined in this Configuration; and, if a name is
+ undefined here,</li>
+ <li>Properties in {@link System#getProperties()}.</li>
+ </ol>
+
+ <p>For example, if a configuration resource contains the following property
+ definitions:
+ <tt><pre>
+ <property>
+ <name>basedir</name>
+ <value>/user/${<i>user.name</i>}</value>
+ </property>
+
+ <property>
+ <name>tempdir</name>
+ <value>${<i>basedir</i>}/tmp</value>
+ </property></pre></tt>
+
+ When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
+ will be resolved to another property in this Configuration, while
+ <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
+ of the System property with that name.
+ By default, warnings will be given to any deprecated configuration
+ parameters and these are suppressible by configuring
+ <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
+ log4j.properties file.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configuration -->
+ <!-- start class org.apache.hadoop.conf.Configuration.DeprecationDelta -->
+ <class name="Configuration.DeprecationDelta" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="DeprecationDelta" type="java.lang.String, java.lang.String, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="DeprecationDelta" type="java.lang.String, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getKey" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getNewKeys" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getCustomMessage" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[A pending addition to the global set of deprecated keys.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configuration.DeprecationDelta -->
+ <!-- start class org.apache.hadoop.conf.Configuration.IntegerRanges -->
+ <class name="Configuration.IntegerRanges" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="java.lang.Iterable"/>
+ <constructor name="IntegerRanges"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="IntegerRanges" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="isIncluded" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="value" type="int"/>
+ <doc>
+ <![CDATA[Is the given value in the set of ranges
+ @param value the value to check
+ @return is the value in the ranges?]]>
+ </doc>
+ </method>
+ <method name="isEmpty" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return true if there are no values in this range, else false.]]>
+ </doc>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="iterator" return="java.util.Iterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[A class that represents a set of positive integer ranges. It parses
+ strings of the form: "2-3,5,7-" where ranges are separated by comma and
+ the lower/upper bounds are separated by dash. Either the lower or upper
+ bound may be omitted meaning all values up to or over. So the string
+ above means 2, 3, 5, and 7, 8, 9, ...]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configuration.IntegerRanges -->
+ <!-- start class org.apache.hadoop.conf.Configured -->
+ <class name="Configured" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.conf.Configurable"/>
+ <constructor name="Configured"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a Configured.]]>
+ </doc>
+ </constructor>
+ <constructor name="Configured" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a Configured.]]>
+ </doc>
+ </constructor>
+ <method name="setConf"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Base class for things that may be configured with a {@link Configuration}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configured -->
+ <!-- start class org.apache.hadoop.conf.ConfServlet.BadFormatException -->
+ <class name="ConfServlet.BadFormatException" extends="java.lang.Exception"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="BadFormatException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ConfServlet.BadFormatException -->
+ <!-- start interface org.apache.hadoop.conf.Reconfigurable -->
+ <interface name="Reconfigurable" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.conf.Configurable"/>
+ <method name="reconfigureProperty" return="java.lang.String"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <param name="newVal" type="java.lang.String"/>
+ <exception name="ReconfigurationException" type="org.apache.hadoop.conf.ReconfigurationException"/>
+ <doc>
+ <![CDATA[Change a configuration property on this object to the value specified.
+
+ Change a configuration property on this object to the value specified
+ and return the previous value that the configuration property was set to
+ (or null if it was not previously set). If newVal is null, set the property
+ to its default value;
+
+ If the property cannot be changed, throw a
+ {@link ReconfigurationException}.]]>
+ </doc>
+ </method>
+ <method name="isPropertyReconfigurable" return="boolean"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Return whether a given property is changeable at run time.
+
+ If isPropertyReconfigurable returns true for a property,
+ then changeConf should not throw an exception when changing
+ this property.]]>
+ </doc>
+ </method>
+ <method name="getReconfigurableProperties" return="java.util.Collection"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return all the properties that can be changed at run time.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Something whose {@link Configuration} can be changed at run time.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.conf.Reconfigurable -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurableBase -->
+ <class name="ReconfigurableBase" extends="org.apache.hadoop.conf.Configured"
+ abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.conf.Reconfigurable"/>
+ <constructor name="ReconfigurableBase"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a ReconfigurableBase.]]>
+ </doc>
+ </constructor>
+ <constructor name="ReconfigurableBase" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a ReconfigurableBase with the {@link Configuration}
+ conf.]]>
+ </doc>
+ </constructor>
+ <method name="setReconfigurationUtil"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ru" type="org.apache.hadoop.conf.ReconfigurationUtil"/>
+ </method>
+ <method name="getChangedProperties" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="newConf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="oldConf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="startReconfigurationTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Start a reconfiguration task to reload configuration in background.]]>
+ </doc>
+ </method>
+ <method name="getReconfigurationTaskStatus" return="org.apache.hadoop.conf.ReconfigurationTaskStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="shutdownReconfigurationTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="reconfigureProperty" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <param name="newVal" type="java.lang.String"/>
+ <exception name="ReconfigurationException" type="org.apache.hadoop.conf.ReconfigurationException"/>
+ <doc>
+ <![CDATA[{@inheritDoc}
+
+ This method makes the change to this objects {@link Configuration}
+ and calls reconfigurePropertyImpl to update internal data structures.
+ This method cannot be overridden, subclasses should instead override
+ reconfigureProperty.]]>
+ </doc>
+ </method>
+ <method name="getReconfigurableProperties" return="java.util.Collection"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[{@inheritDoc}
+
+ Subclasses must override this.]]>
+ </doc>
+ </method>
+ <method name="isPropertyReconfigurable" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <doc>
+ <![CDATA[{@inheritDoc}
+
+ Subclasses may wish to override this with a more efficient implementation.]]>
+ </doc>
+ </method>
+ <method name="reconfigurePropertyImpl"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <param name="newVal" type="java.lang.String"/>
+ <exception name="ReconfigurationException" type="org.apache.hadoop.conf.ReconfigurationException"/>
+ <doc>
+ <![CDATA[Change a configuration property.
+
+ Subclasses must override this. This method applies the change to
+ all internal data structures derived from the configuration property
+ that is being changed. If this object owns other Reconfigurable objects
+ reconfigureProperty should be called recursively to make sure that
+ to make sure that the configuration of these objects is updated.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Utility base class for implementing the Reconfigurable interface.
+
+ Subclasses should override reconfigurePropertyImpl to change individual
+ properties and getReconfigurableProperties to get all properties that
+ can be changed at run time.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ReconfigurableBase -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurationException -->
+ <class name="ReconfigurationException" extends="java.lang.Exception"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ReconfigurationException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Create a new instance of {@link ReconfigurationException}.]]>
+ </doc>
+ </constructor>
+ <constructor name="ReconfigurationException" type="java.lang.String, java.lang.String, java.lang.String, java.lang.Throwable"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Create a new instance of {@link ReconfigurationException}.]]>
+ </doc>
+ </constructor>
+ <constructor name="ReconfigurationException" type="java.lang.String, java.lang.String, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Create a new instance of {@link ReconfigurationException}.]]>
+ </doc>
+ </constructor>
+ <method name="getProperty" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get property that cannot be changed.]]>
+ </doc>
+ </method>
+ <method name="getNewValue" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get value to which property was supposed to be changed.]]>
+ </doc>
+ </method>
+ <method name="getOldValue" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get old value of property that cannot be changed.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Exception indicating that configuration property cannot be changed
+ at run time.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ReconfigurationException -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurationServlet -->
+ <class name="ReconfigurationServlet" extends="javax.servlet.http.HttpServlet"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ReconfigurationServlet"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="init"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="ServletException" type="javax.servlet.ServletException"/>
+ </method>
+ <method name="doGet"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="req" type="javax.servlet.http.HttpServletRequest"/>
+ <param name="resp" type="javax.servlet.http.HttpServletResponse"/>
+ <exception name="ServletException" type="javax.servlet.ServletException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="doPost"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="req" type="javax.servlet.http.HttpServletRequest"/>
+ <param name="resp" type="javax.servlet.http.HttpSer
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-5544.
TestNodeBlacklistingOnAMFailures fails on trunk. Contributed by Sunil G.
Posted by cd...@apache.org.
YARN-5544. TestNodeBlacklistingOnAMFailures fails on trunk. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d5997d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d5997d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d5997d2
Branch: refs/heads/HDFS-9806
Commit: 0d5997d2b98eb89e72828dfcd78f02aa4e7e1e67
Parents: 8cc4a67
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Aug 23 10:33:28 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Aug 23 14:37:39 2016 +0530
----------------------------------------------------------------------
.../TestNodeBlacklistingOnAMFailures.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5997d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
index ef6d43b..7a24b7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
@@ -68,6 +68,9 @@ public class TestNodeBlacklistingOnAMFailures {
MockRM rm = startRM(conf, dispatcher);
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+ // Register 5 nodes, so that we can blacklist atleast one if AM container
+ // is failed. As per calculation it will be like, 5nodes * 0.2 (default)=1.
+ // First register 2 nodes, and after AM lauched register 3 more nodes.
MockNM nm1 =
new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService());
nm1.registerNode();
@@ -93,6 +96,19 @@ public class TestNodeBlacklistingOnAMFailures {
otherNode = nm1;
}
+ // register 3 nodes now
+ MockNM nm3 =
+ new MockNM("127.0.0.3:2345", 8000, rm.getResourceTrackerService());
+ nm3.registerNode();
+
+ MockNM nm4 =
+ new MockNM("127.0.0.4:2345", 8000, rm.getResourceTrackerService());
+ nm4.registerNode();
+
+ MockNM nm5 =
+ new MockNM("127.0.0.5:2345", 8000, rm.getResourceTrackerService());
+ nm5.registerNode();
+
// Set the exist status to INVALID so that we can verify that the system
// automatically blacklisting the node
makeAMContainerExit(rm, amContainerId, currentNode,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HADOOP-13252. Tune S3A provider
plugin mechanism. Contributed by Steve Loughran.
Posted by cd...@apache.org.
HADOOP-13252. Tune S3A provider plugin mechanism. Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/763f0497
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/763f0497
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/763f0497
Branch: refs/heads/HDFS-9806
Commit: 763f0497bb996e331e40caed9ca0af966f5b3fac
Parents: 03a9343
Author: Chris Nauroth <cn...@apache.org>
Authored: Fri Aug 19 10:48:10 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Fri Aug 19 10:48:10 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 19 +-
.../fs/FileContextMainOperationsBaseTest.java | 8 +-
.../apache/hadoop/fs/FileContextUtilBase.java | 4 +-
.../fs/s3a/AWSCredentialProviderList.java | 188 +++++++++
.../fs/s3a/AnonymousAWSCredentialsProvider.java | 4 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 6 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 56 +--
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 139 ++++++-
.../fs/s3a/SimpleAWSCredentialsProvider.java | 86 +++++
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 32 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 381 +++++++++++++++----
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 98 ++++-
.../fs/s3a/TestS3ATemporaryCredentials.java | 4 +-
.../org/apache/hadoop/fs/s3a/yarn/TestS3A.java | 4 +-
14 files changed, 843 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index dd8f9af..569d894 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -877,22 +877,27 @@
<property>
<name>fs.s3a.aws.credentials.provider</name>
<description>
- Class name of a credentials provider that implements
- com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys
- or another authentication mechanism. The specified class must provide an
- accessible constructor accepting java.net.URI and
- org.apache.hadoop.conf.Configuration, or an accessible default constructor.
+ Comma-separated class names of credential provider classes which implement
+ com.amazonaws.auth.AWSCredentialsProvider.
+
+ These are loaded and queried in sequence for a valid set of credentials.
+ Each listed class must provide either an accessible constructor accepting
+ java.net.URI and org.apache.hadoop.conf.Configuration, or an accessible
+ default constructor.
+
Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
anonymous access to a publicly accessible S3 bucket without any credentials.
Please note that allowing anonymous access to an S3 bucket compromises
- security and therefore is unsuitable for most use cases. It can be useful
+ security and therefore is unsuitable for most use cases. It can be useful
for accessing public data sets without requiring AWS credentials.
</description>
</property>
<property>
<name>fs.s3a.session.token</name>
- <description>The session token used with temporary credentials. Used only with provider org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider.</description>
+ <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
+ as one of the providers.
+ </description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index 78b40b5..5f9151a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -109,9 +109,11 @@ public abstract class FileContextMainOperationsBaseTest {
@After
public void tearDown() throws Exception {
- boolean del = fc.delete(new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")), true);
- assertTrue(del);
- fc.delete(localFsRootPath, true);
+ if (fc != null) {
+ boolean del = fc.delete(new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")), true);
+ assertTrue(del);
+ fc.delete(localFsRootPath, true);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java
index c473bdf..bebf4bf 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextUtilBase.java
@@ -63,7 +63,9 @@ public abstract class FileContextUtilBase {
@After
public void tearDown() throws Exception {
- fc.delete(fileContextTestHelper.getTestRootPath(fc), true);
+ if (fc != null) {
+ fc.delete(fileContextTestHelper.getTestRootPath(fc), true);
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
new file mode 100644
index 0000000..cee3269
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AnonymousAWSCredentials;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * A list of providers.
+ *
+ * This is similar to the AWS SDK {@code AWSCredentialsProviderChain},
+ * except that:
+ * <ol>
+ * <li>Allows extra providers to be added dynamically.</li>
+ * <li>If any provider in the chain throws an exception other than
+ * an {@link AmazonClientException}, that is rethrown, rather than
+ * swallowed.</li>
+ * <li>Has some more diagnostics.</li>
+ * <li>On failure, the last AmazonClientException raised is rethrown.</li>
+ * <li>Special handling of {@link AnonymousAWSCredentials}.</li>
+ * </ol>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class AWSCredentialProviderList implements AWSCredentialsProvider {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ AWSCredentialProviderList.class);
+ public static final String NO_AWS_CREDENTIAL_PROVIDERS
+ = "No AWS Credential Providers";
+
+ private final List<AWSCredentialsProvider> providers = new ArrayList<>(1);
+ private boolean reuseLastProvider = true;
+ private AWSCredentialsProvider lastProvider;
+
+ /**
+ * Empty instance. This is not ready to be used.
+ */
+ public AWSCredentialProviderList() {
+ }
+
+ /**
+ * Create with an initial list of providers.
+ * @param providers provider list.
+ */
+ public AWSCredentialProviderList(
+ Collection<AWSCredentialsProvider> providers) {
+ this.providers.addAll(providers);
+ }
+
+ /**
+ * Add a new provider.
+ * @param p provider
+ */
+ public void add(AWSCredentialsProvider p) {
+ providers.add(p);
+ }
+
+ /**
+ * Reuse the last provider?
+ * @param reuseLastProvider flag to indicate the last provider should
+ * be re-used
+ */
+ public void setReuseLastProvider(boolean reuseLastProvider) {
+ this.reuseLastProvider = reuseLastProvider;
+ }
+
+ /**
+ * query the {@link #reuseLastProvider} flag.
+ * @return the current flag state.
+ */
+ public boolean isReuseLastProvider() {
+ return reuseLastProvider;
+ }
+
+ /**
+ * Refresh all child entries.
+ */
+ @Override
+ public void refresh() {
+ for (AWSCredentialsProvider provider : providers) {
+ provider.refresh();
+ }
+ }
+
+ /**
+ * Iterate through the list of providers, to find one with credentials.
+ * If {@link #reuseLastProvider} is true, then it is re-used.
+ * @return a set of credentials (possibly anonymous), for authenticating.
+ */
+ @Override
+ public AWSCredentials getCredentials() {
+ checkNotEmpty();
+ if (reuseLastProvider && lastProvider != null) {
+ return lastProvider.getCredentials();
+ }
+
+ AmazonClientException lastException = null;
+ for (AWSCredentialsProvider provider : providers) {
+ try {
+ AWSCredentials credentials = provider.getCredentials();
+ if ((credentials.getAWSAccessKeyId() != null &&
+ credentials.getAWSSecretKey() != null)
+ || (credentials instanceof AnonymousAWSCredentials)) {
+ lastProvider = provider;
+ LOG.debug("Using credentials from {}", provider);
+ return credentials;
+ }
+ } catch (AmazonClientException e) {
+ lastException = e;
+ LOG.debug("No credentials provided by {}: {}",
+ provider, e.toString(), e);
+ }
+ }
+
+ // no providers had any credentials. Rethrow the last exception
+ // or create a new one.
+ String message = "No AWS Credentials provided by "
+ + listProviderNames();
+ if (lastException != null) {
+ message += ": " + lastException;
+ }
+ throw new AmazonClientException(message, lastException);
+
+ }
+
+ /**
+ * Verify that the provider list is not empty.
+ * @throws AmazonClientException if there are no providers.
+ */
+ public void checkNotEmpty() {
+ if (providers.isEmpty()) {
+ throw new AmazonClientException(NO_AWS_CREDENTIAL_PROVIDERS);
+ }
+ }
+
+ /**
+ * List all the providers' names.
+ * @return a list of names, separated by spaces (with a trailing one).
+ * If there are no providers, "" is returned.
+ */
+ public String listProviderNames() {
+ StringBuilder sb = new StringBuilder(providers.size() * 32);
+ for (AWSCredentialsProvider provider : providers) {
+ sb.append(provider.getClass().getSimpleName());
+ sb.append(' ');
+ }
+ return sb.toString();
+ }
+
+ /**
+ * The string value is this class name and the string values of nested
+ * providers.
+ * @return a string value for debugging.
+ */
+ @Override
+ public String toString() {
+ return "AWSCredentialProviderList: " +
+ StringUtils.join(providers, " ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
index 2c863fc..0cb9fd4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java
@@ -38,6 +38,10 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Private
@InterfaceStability.Stable
public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider {
+
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider";
+
public AWSCredentials getCredentials() {
return new AnonymousAWSCredentials();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
index 61be43f..b1899e2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
@@ -29,16 +29,16 @@ import org.apache.hadoop.classification.InterfaceStability;
* BasicAWSCredentialsProvider supports static configuration of access key ID
* and secret access key for use with the AWS SDK.
*
- * Please note that users may reference this class name from configuration
- * property fs.s3a.aws.credentials.provider. Therefore, changing the class name
- * would be a backward-incompatible change.
*/
@InterfaceAudience.Private
@InterfaceStability.Stable
public class BasicAWSCredentialsProvider implements AWSCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.BasicAWSCredentialsProvider";
private final String accessKey;
private final String secretKey;
+
public BasicAWSCredentialsProvider(String accessKey, String secretKey) {
this.accessKey = accessKey;
this.secretKey = secretKey;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 1cf2631..7231349 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -32,14 +32,11 @@ import java.util.concurrent.ExecutorService;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
-import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSCredentialsProviderChain;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.services.s3.model.AmazonS3Exception;
@@ -151,7 +148,7 @@ public class S3AFileSystem extends FileSystem {
bucket = name.getHost();
AWSCredentialsProvider credentials =
- getAWSCredentialsProvider(name, conf);
+ createAWSCredentialProviderSet(name, conf, uri);
ClientConfiguration awsConf = new ClientConfiguration();
awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
@@ -411,57 +408,6 @@ public class S3AFileSystem extends FileSystem {
}
/**
- * Create the standard credential provider, or load in one explicitly
- * identified in the configuration.
- * @param binding the S3 binding/bucket.
- * @param conf configuration
- * @return a credential provider
- * @throws IOException on any problem. Class construction issues may be
- * nested inside the IOE.
- */
- private AWSCredentialsProvider getAWSCredentialsProvider(URI binding,
- Configuration conf) throws IOException {
- AWSCredentialsProvider credentials;
-
- String className = conf.getTrimmed(AWS_CREDENTIALS_PROVIDER);
- if (StringUtils.isEmpty(className)) {
- S3xLoginHelper.Login creds = getAWSAccessKeys(binding, conf);
- credentials = new AWSCredentialsProviderChain(
- new BasicAWSCredentialsProvider(
- creds.getUser(), creds.getPassword()),
- new InstanceProfileCredentialsProvider(),
- new EnvironmentVariableCredentialsProvider());
-
- } else {
- try {
- LOG.debug("Credential provider class is {}", className);
- Class<?> credClass = Class.forName(className);
- try {
- credentials =
- (AWSCredentialsProvider)credClass.getDeclaredConstructor(
- URI.class, Configuration.class).newInstance(this.uri, conf);
- } catch (NoSuchMethodException | SecurityException e) {
- credentials =
- (AWSCredentialsProvider)credClass.getDeclaredConstructor()
- .newInstance();
- }
- } catch (ClassNotFoundException e) {
- throw new IOException(className + " not found.", e);
- } catch (NoSuchMethodException | SecurityException e) {
- throw new IOException(String.format("%s constructor exception. A "
- + "class specified in %s must provide an accessible constructor "
- + "accepting URI and Configuration, or an accessible default "
- + "constructor.", className, AWS_CREDENTIALS_PROVIDER), e);
- } catch (ReflectiveOperationException | IllegalArgumentException e) {
- throw new IOException(className + " instantiation exception.", e);
- }
- LOG.debug("Using {} for {}.", credentials, this.uri);
- }
-
- return credentials;
- }
-
- /**
* Return the protocol scheme for the FileSystem.
*
* @return "s3a"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 3d48fcd..699676d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.fs.s3a;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.apache.commons.lang.StringUtils;
@@ -29,6 +32,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
import java.io.EOFException;
import java.io.FileNotFoundException;
@@ -40,6 +44,7 @@ import java.util.Map;
import java.util.concurrent.ExecutionException;
import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
/**
@@ -49,6 +54,14 @@ import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
@InterfaceStability.Evolving
public final class S3AUtils {
+ /** Reuse the S3AFileSystem log. */
+ private static final Logger LOG = S3AFileSystem.LOG;
+ static final String CONSTRUCTOR_EXCEPTION = "constructor exception";
+ static final String INSTANTIATION_EXCEPTION
+ = "instantiation exception";
+ static final String NOT_AWS_PROVIDER =
+ "does not implement AWSCredentialsProvider";
+
private S3AUtils() {
}
@@ -244,6 +257,85 @@ public final class S3AUtils {
}
/**
+ * Create the AWS credentials from the providers and the URI.
+ * @param binding Binding URI, may contain user:pass login details
+ * @param conf filesystem configuration
+ * @param fsURI fS URI \u2014after any login details have been stripped.
+ * @return a credentials provider list
+ * @throws IOException Problems loading the providers (including reading
+ * secrets from credential files).
+ */
+ public static AWSCredentialProviderList createAWSCredentialProviderSet(
+ URI binding,
+ Configuration conf,
+ URI fsURI) throws IOException {
+ AWSCredentialProviderList credentials = new AWSCredentialProviderList();
+
+ Class<?>[] awsClasses;
+ try {
+ awsClasses = conf.getClasses(AWS_CREDENTIALS_PROVIDER);
+ } catch (RuntimeException e) {
+ Throwable c = e.getCause() != null ? e.getCause() : e;
+ throw new IOException("From option " + AWS_CREDENTIALS_PROVIDER +
+ ' ' + c, c);
+ }
+ if (awsClasses.length == 0) {
+ S3xLoginHelper.Login creds = getAWSAccessKeys(binding, conf);
+ credentials.add(new BasicAWSCredentialsProvider(
+ creds.getUser(), creds.getPassword()));
+ credentials.add(new EnvironmentVariableCredentialsProvider());
+ credentials.add(new InstanceProfileCredentialsProvider());
+ } else {
+ for (Class<?> aClass : awsClasses) {
+ credentials.add(createAWSCredentialProvider(conf,
+ aClass,
+ fsURI));
+ }
+ }
+ return credentials;
+ }
+
+ /**
+ * Create an AWS credential provider.
+ * @param conf configuration
+ * @param credClass credential class
+ * @param uri URI of the FS
+ * @return the instantiated class
+ * @throws IOException on any instantiation failure.
+ */
+ static AWSCredentialsProvider createAWSCredentialProvider(
+ Configuration conf,
+ Class<?> credClass,
+ URI uri) throws IOException {
+ AWSCredentialsProvider credentials;
+ String className = credClass.getName();
+ if (!AWSCredentialsProvider.class.isAssignableFrom(credClass)) {
+ throw new IOException("Class " + credClass + " " + NOT_AWS_PROVIDER);
+ }
+ try {
+ LOG.debug("Credential provider class is {}", className);
+ try {
+ credentials =
+ (AWSCredentialsProvider) credClass.getDeclaredConstructor(
+ URI.class, Configuration.class).newInstance(uri, conf);
+ } catch (NoSuchMethodException | SecurityException e) {
+ credentials =
+ (AWSCredentialsProvider) credClass.getDeclaredConstructor()
+ .newInstance();
+ }
+ } catch (NoSuchMethodException | SecurityException e) {
+ throw new IOException(String.format("%s " + CONSTRUCTOR_EXCEPTION
+ +". A class specified in %s must provide an accessible constructor "
+ + "accepting URI and Configuration, or an accessible default "
+ + "constructor.", className, AWS_CREDENTIALS_PROVIDER), e);
+ } catch (ReflectiveOperationException | IllegalArgumentException e) {
+ throw new IOException(className + " " + INSTANTIATION_EXCEPTION +".", e);
+ }
+ LOG.debug("Using {} for {}.", credentials, uri);
+ return credentials;
+ }
+
+ /**
* Return the access key and secret for S3 API use.
* Credentials may exist in configuration, within credential providers
* or indicated in the UserInfo of the name URI param.
@@ -263,21 +355,40 @@ public final class S3AUtils {
return new S3xLoginHelper.Login(accessKey, secretKey);
}
- private static String getPassword(Configuration conf, String key, String val)
+ /**
+ * Get a password from a configuration, or, if a value is passed in,
+ * pick that up instead.
+ * @param conf configuration
+ * @param key key to look up
+ * @param val current value: if non empty this is used instead of
+ * querying the configuration.
+ * @return a password or "".
+ * @throws IOException on any problem
+ */
+ static String getPassword(Configuration conf, String key, String val)
throws IOException {
- if (StringUtils.isEmpty(val)) {
- try {
- final char[] pass = conf.getPassword(key);
- if (pass != null) {
- return (new String(pass)).trim();
- } else {
- return "";
- }
- } catch (IOException ioe) {
- throw new IOException("Cannot find password option " + key, ioe);
- }
- } else {
- return val;
+ return StringUtils.isEmpty(val)
+ ? lookupPassword(conf, key, "")
+ : val;
+ }
+
+ /**
+ * Get a password from a configuration/configured credential providers.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal value to return if there is no password
+ * @return a password or the value in {@code defVal}
+ * @throws IOException on any problem
+ */
+ static String lookupPassword(Configuration conf, String key, String defVal)
+ throws IOException {
+ try {
+ final char[] pass = conf.getPassword(key);
+ return pass != null ?
+ new String(pass).trim()
+ : defVal;
+ } catch (IOException ioe) {
+ throw new IOException("Cannot find password option " + key, ioe);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
new file mode 100644
index 0000000..13c139d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
+
+import java.io.IOException;
+import java.net.URI;
+
+import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
+import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY;
+
+/**
+ * Support simple credentials for authenticating with AWS.
+ * Keys generated in URLs are not supported.
+ *
+ * Please note that users may reference this class name from configuration
+ * property fs.s3a.aws.credentials.provider. Therefore, changing the class name
+ * would be a backward-incompatible change.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
+
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider";
+ private String accessKey;
+ private String secretKey;
+ private IOException lookupIOE;
+
+ public SimpleAWSCredentialsProvider(URI uri, Configuration conf) {
+ try {
+ Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, S3AFileSystem.class);
+ this.accessKey = S3AUtils.lookupPassword(c, ACCESS_KEY, null);
+ this.secretKey = S3AUtils.lookupPassword(c, SECRET_KEY, null);
+ } catch (IOException e) {
+ lookupIOE = e;
+ }
+ }
+
+ public AWSCredentials getCredentials() {
+ if (lookupIOE != null) {
+ // propagate any initialization problem
+ throw new CredentialInitializationException(lookupIOE.toString(),
+ lookupIOE);
+ }
+ if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)) {
+ return new BasicAWSCredentials(accessKey, secretKey);
+ }
+ throw new CredentialInitializationException(
+ "Access key, secret key or session token is unset");
+ }
+
+ @Override
+ public void refresh() {}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
index 190f7bc..883ae86 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
@@ -23,34 +23,52 @@ import com.amazonaws.auth.BasicSessionCredentials;
import com.amazonaws.auth.AWSCredentials;
import org.apache.commons.lang.StringUtils;
+import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
import static org.apache.hadoop.fs.s3a.Constants.*;
/**
* Support session credentials for authenticating with AWS.
+ *
+ * Please note that users may reference this class name from configuration
+ * property fs.s3a.aws.credentials.provider. Therefore, changing the class name
+ * would be a backward-incompatible change.
*/
-@InterfaceAudience.Private
+@InterfaceAudience.Public
@InterfaceStability.Stable
public class TemporaryAWSCredentialsProvider implements AWSCredentialsProvider {
public static final String NAME
= "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
- private final String accessKey;
- private final String secretKey;
- private final String sessionToken;
+ private String accessKey;
+ private String secretKey;
+ private String sessionToken;
+ private IOException lookupIOE;
public TemporaryAWSCredentialsProvider(URI uri, Configuration conf) {
- this.accessKey = conf.get(ACCESS_KEY, null);
- this.secretKey = conf.get(SECRET_KEY, null);
- this.sessionToken = conf.get(SESSION_TOKEN, null);
+ try {
+ Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, S3AFileSystem.class);
+ this.accessKey = S3AUtils.lookupPassword(c, ACCESS_KEY, null);
+ this.secretKey = S3AUtils.lookupPassword(c, SECRET_KEY, null);
+ this.sessionToken = S3AUtils.lookupPassword(c, SESSION_TOKEN, null);
+ } catch (IOException e) {
+ lookupIOE = e;
+ }
}
public AWSCredentials getCredentials() {
+ if (lookupIOE != null) {
+ // propagate any initialization problem
+ throw new CredentialInitializationException(lookupIOE.toString(),
+ lookupIOE);
+ }
if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)
&& !StringUtils.isEmpty(sessionToken)) {
return new BasicSessionCredentials(accessKey, secretKey, sessionToken);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index d459fb5..78c9f12 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -276,138 +276,334 @@ one where patches for functionality and performance are very welcome.
* `httpclient` jar.
* Jackson `jackson-core`, `jackson-annotations`, `jackson-databind` jars.
+### S3A Authentication methods
+
+S3A supports multiple authentication mechanisms, and can be configured as to
+which mechanisms to use, and the order to use them. Custom implementations
+of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
+
### Authentication properties
<property>
<name>fs.s3a.access.key</name>
- <description>AWS access key ID. Omit for IAM role-based or provider-based authentication.</description>
+ <description>AWS access key ID.
+ Omit for IAM role-based or provider-based authentication.</description>
</property>
<property>
<name>fs.s3a.secret.key</name>
- <description>AWS secret key. Omit for IAM role-based or provider-based authentication.</description>
+ <description>AWS secret key.
+ Omit for IAM role-based or provider-based authentication.</description>
</property>
<property>
<name>fs.s3a.aws.credentials.provider</name>
<description>
- Class name of a credentials provider that implements
- com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys
- or another authentication mechanism. The specified class must provide an
- accessible constructor accepting java.net.URI and
- org.apache.hadoop.conf.Configuration, or an accessible default constructor.
+ Comma-separated class names of credential provider classes which implement
+ com.amazonaws.auth.AWSCredentialsProvider.
+
+ These are loaded and queried in sequence for a valid set of credentials.
+ Each listed class must provide either an accessible constructor accepting
+ java.net.URI and org.apache.hadoop.conf.Configuration, or an accessible
+ default constructor.
+
Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
anonymous access to a publicly accessible S3 bucket without any credentials.
Please note that allowing anonymous access to an S3 bucket compromises
- security and therefore is unsuitable for most use cases. It can be useful
+ security and therefore is unsuitable for most use cases. It can be useful
for accessing public data sets without requiring AWS credentials.
</description>
</property>
<property>
<name>fs.s3a.session.token</name>
- <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider as the providers.</description>
+ <description>
+ Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider
+ as one of the providers.
+ </description>
</property>
-#### Authentication methods
+
+#### Authenticating via environment variables
+
+S3A supports configuration via [the standard AWS environment variables](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment).
+
+The core environment variables are for the access key and associated secret:
+
+```
+export AWS_ACCESS_KEY_ID=my.aws.key
+export AWS_SECRET_ACCESS_KEY=my.secret.key
+```
+
+If the environment variable `AWS_SESSION_TOKEN` is set, session authentication
+using "Temporary Security Credentials" is enabled; the Key ID and secret key
+must be set to the credentials for that specific sesssion.
+
+```
+export AWS_SESSION_TOKEN=SECRET-SESSION-TOKEN
+export AWS_ACCESS_KEY_ID=SESSION-ACCESS-KEY
+export AWS_SECRET_ACCESS_KEY=SESSION-SECRET-KEY
+```
+
+These environment variables can be used to set the authentication credentials
+instead of properties in the Hadoop configuration.
+
+*Important:*
+These environment variables are not propagated from client to server when
+YARN applications are launched. That is: having the AWS environment variables
+set when an application is launched will not permit the launched application
+to access S3 resources. The environment variables must (somehow) be set
+on the hosts/processes where the work is executed.
+
+
+#### Changing Authentication Providers
The standard way to authenticate is with an access key and secret key using the
-properties above. You can also avoid configuring credentials if the EC2
-instances in your cluster are configured with IAM instance profiles that grant
-the appropriate S3 access.
-
-A temporary set of credentials can also be obtained from Amazon STS; these
-consist of an access key, a secret key, and a session token. To use these
-temporary credentials you must include the `aws-java-sdk-sts` JAR in your
-classpath (consult the POM for the current version) and set the
-`TemporaryAWSCredentialsProvider` class as the provider. The session key
-must be set in the property `fs.s3a.session.token` \u2014and the access and secret
-key properties to those of this temporary session.
+properties in the configuration file.
+
+The S3A client follows the following authentication chain:
+
+1. If login details were provided in the filesystem URI, a warning is printed
+and then the username and password extracted for the AWS key and secret respectively.
+1. The `fs.s3a.access.key` and `fs.s3a.secret.key` are looked for in the Hadoop
+XML configuration.
+1. The [AWS environment variables](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment),
+are then looked for.
+1. An attempt is made to query the Amazon EC2 Instance Metadata Service to
+ retrieve credentials published to EC2 VMs.
+
+S3A can be configured to obtain client authentication providers from classes
+which integrate with the AWS SDK by implementing the `com.amazonaws.auth.AWSCredentialsProvider`
+Interface. This is done by listing the implementation classes, in order of
+preference, in the configuration option `fs.s3a.aws.credentials.provider`.
+
+*Important*: AWS Credential Providers are distinct from _Hadoop Credential Providers_.
+As will be covered later, Hadoop Credential Providers allow passwords and other secrets
+to be stored and transferred more securely than in XML configuration files.
+AWS Credential Providers are classes which can be used by the Amazon AWS SDK to
+obtain an AWS login from a different source in the system, including environment
+variables, JVM properties and configuration files.
+
+There are three AWS Credential Providers inside the `hadoop-aws` JAR:
+
+| classname | description |
+|-----------|-------------|
+| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials |
+| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials |
+| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login |
+
+There are also many in the Amazon SDKs, in particular two which are automatically
+set up in the authentication chain:
+
+| classname | description |
+|-----------|-------------|
+| `com.amazonaws.auth.InstanceProfileCredentialsProvider`| EC2 Metadata Credentials |
+| `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables |
+
+
+*Session Credentials with `TemporaryAWSCredentialsProvider`*
+
+[Temporary Security Credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
+can be obtained from the Amazon Security Token Service; these
+consist of an access key, a secret key, and a session token.
+
+To authenticate with these:
+
+1. Declare `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider` as the
+provider.
+1. Set the session key in the property `fs.s3a.session.token`,
+and the access and secret key properties to those of this temporary session.
+
+Example:
+
+```xml
+<property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider</value>
+</property>
+
+<property>
+ <name>fs.s3a.access.key</name>
+ <value>SESSION-ACCESS-KEY</value>
+</property>
+
+<property>
+ <name>fs.s3a.secret.key</name>
+ <value>SESSION-SECRET-KEY</value>
+</property>
+
+<property>
+ <name>fs.s3a.session.token</name>
+ <value>SECRET-SESSION-TOKEN</value>
+</property>
+```
+
+The lifetime of session credentials are fixed when the credentials
+are issued; once they expire the application will no longer be able to
+authenticate to AWS.
+
+*Anonymous Login with `AnonymousAWSCredentialsProvider`*
+
+Specifying `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider` allows
+anonymous access to a publicly accessible S3 bucket without any credentials.
+It can be useful for accessing public data sets without requiring AWS credentials.
+
+```xml
+<property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider</value>
+</property>
+```
+
+Once this is done, there's no need to supply any credentials
+in the Hadoop configuration or via environment variables.
+
+This option can be used to verify that an object store does
+not permit unauthenticated access: that is, if an attempt to list
+a bucket is made using the anonymous credentials, it should fail \u2014unless
+explicitly opened up for broader access.
+
+```bash
+hadoop fs -ls \
+ -D fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider \
+ s3a://landsat-pds/
+```
+
+1. Allowing anonymous access to an S3 bucket compromises
+security and therefore is unsuitable for most use cases.
+
+1. If a list of credential providers is given in `fs.s3a.aws.credentials.provider`,
+then the Anonymous Credential provider *must* come last. If not, credential
+providers listed after it will be ignored.
+
+*Simple name/secret credentials with `SimpleAWSCredentialsProvider`*
+
+This is is the standard credential provider, which
+supports the secret key in `fs.s3a.access.key` and token in `fs.s3a.secret.key`
+values. It does not support authentication with logins credentials declared
+in the URLs.
<property>
<name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider</value>
+ <value>org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider</value>
</property>
- <property>
- <name>fs.s3a.access.key</name>
- <value>SESSION-ACCESS-KEY</value>
- </property>
+Apart from its lack of support of user:password details being included in filesystem
+URLs (a dangerous practise that is strongly discouraged), this provider acts
+exactly at the basic authenticator used in the default authentication chain.
- <property>
- <name>fs.s3a.secret.key</name>
- <value>SESSION-SECRET-KEY</value>
- </property>
+This means that the default S3A authentication chain can be defined as
<property>
- <name>fs.s3a.session.token</name>
- <value>SECRET-SESSION-TOKEN</value>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <value>
+ org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
+ com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
+ com.amazonaws.auth.InstanceProfileCredentialsProvider
+ </value>
</property>
-#### Protecting the AWS Credentials in S3A
+
+#### Protecting the AWS Credentials
To protect the access/secret keys from prying eyes, it is recommended that you
use either IAM role-based authentication (such as EC2 instance profile) or
the credential provider framework securely storing them and accessing them
through configuration. The following describes using the latter for AWS
-credentials in S3AFileSystem.
+credentials in the S3A FileSystem.
-For additional reading on the credential provider API see:
-[Credential Provider API](../../../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html).
-#### Authenticating via environment variables
+##### Storing secrets with Hadoop Credential Providers
-S3A supports configuration via [the standard AWS environment variables](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment).
+The Hadoop Credential Provider Framework allows secure "Credential Providers"
+to keep secrets outside Hadoop configuration files, storing them in encrypted
+files in local or Hadoop filesystems, and including them in requests.
-The core environment variables are for the access key and associated secret:
+The S3A configuration options with sensitive data
+(`fs.s3a.secret.key`, `fs.s3a.access.key` and `fs.s3a.session.token`) can
+have their data saved to a binary file stored, with the values being read in
+when the S3A filesystem URL is used for data access. The reference to this
+credential provider is all that is passed as a direct configuration option.
-```
-export AWS_ACCESS_KEY_ID=my.aws.key
-export AWS_SECRET_ACCESS_KEY=my.secret.key
-```
+For additional reading on the Hadoop Credential Provider API see:
+[Credential Provider API](../../../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html).
-These environment variables can be used to set the authentication credentials
-instead of properties in the Hadoop configuration. *Important:* these
-environment variables are not propagated from client to server when
-YARN applications are launched. That is: having the AWS environment variables
-set when an application is launched will not permit the launched application
-to access S3 resources. The environment variables must (somehow) be set
-on the hosts/processes where the work is executed.
-##### End to End Steps for Distcp and S3 with Credential Providers
+###### Create a credential file
-###### provision
+A credential file can be created on any Hadoop filesystem; when creating one on HDFS or
+a Unix filesystem the permissions are automatically set to keep the file
+private to the reader \u2014though as directory permissions are not touched,
+users should verify that the directory containing the file is readable only by
+the current user.
-```
-% hadoop credential create fs.s3a.access.key -value 123
- -provider localjceks://file/home/lmccay/aws.jceks
-```
-```
-% hadoop credential create fs.s3a.secret.key -value 456
- -provider localjceks://file/home/lmccay/aws.jceks
+```bash
+hadoop credential create fs.s3a.access.key -value 123 \
+ -provider jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks
+
+hadoop credential create fs.s3a.secret.key -value 456 \
+ -provider jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks
```
-###### configure core-site.xml or command line system property
+A credential file can be listed, to see what entries are kept inside it
+```bash
+hadoop credential list -provider jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks
+
+Listing aliases for CredentialProvider: jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks
+fs.s3a.secret.key
+fs.s3a.access.key
```
+At this point, the credentials are ready for use.
+
+###### Configure the `hadoop.security.credential.provider.path` property
+
+The URL to the provider must be set in the configuration property
+`hadoop.security.credential.provider.path`, either on the command line or
+in XML configuration files.
+
+```xml
<property>
<name>hadoop.security.credential.provider.path</name>
- <value>localjceks://file/home/lmccay/aws.jceks</value>
+ <value>jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks</value>
<description>Path to interrogate for protected credentials.</description>
</property>
```
-###### distcp
+
+Because this property only supplies the path to the secrets file, the configuration
+option itself is no longer a sensitive item.
+
+###### Using the credentials
+
+Once the provider is set in the Hadoop configuration, hadoop commands
+work exactly as if the secrets were in an XML file.
+
+```bash
+
+hadoop distcp \
+ hdfs://nn1.example.com:9001/user/backup/007020615 s3a://glacier1/
+
+hadoop fs -ls s3a://glacier1/
```
-% hadoop distcp
- [-D hadoop.security.credential.provider.path=localjceks://file/home/lmccay/aws.jceks]
- hdfs://hostname:9001/user/lmccay/007020615 s3a://lmccay/
+
+The path to the provider can also be set on the command line:
+
+```bash
+hadoop distcp \
+ -D hadoop.security.credential.provider.path=jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks \
+ hdfs://nn1.example.com:9001/user/backup/007020615 s3a://glacier1/
+
+hadoop fs \
+ -D hadoop.security.credential.provider.path=jceks://hdfs@nn1.example.com:9001/user/backup/s3.jceks \
+ -ls s3a://glacier1/
+
```
-NOTE: You may optionally add the provider path property to the distcp command line instead of
-added job specific configuration to a generic core�site.xml. The square brackets above illustrate
-this capability.
+Because the provider path is not itself a sensitive secret, there is no risk
+from placing its declaration on the command line.
+
### Other properties
@@ -824,7 +1020,7 @@ the classpath.
### `ClassNotFoundException: com.amazonaws.services.s3.AmazonS3Client`
(or other `com.amazonaws` class.)
-`
+
This means that one or more of the `aws-*-sdk` JARs are missing. Add them.
### Missing method in AWS class
@@ -851,21 +1047,15 @@ classpath. All Jackson JARs on the classpath *must* be of the same version.
### Authentication failure
-One authentication problem is caused by classpath mismatch; see the joda time
-issue above.
-
-Otherwise, the general cause is: you have the wrong credentials \u2014or somehow
+The general cause is: you have the wrong credentials \u2014or somehow
the credentials were not readable on the host attempting to read or write
the S3 Bucket.
-There's not much that Hadoop can do/does for diagnostics here,
-though enabling debug logging for the package `org.apache.hadoop.fs.s3a`
-can help.
+There's not much that Hadoop can do for diagnostics here.
+Enabling debug logging for the package `org.apache.hadoop.fs.s3a`
+can help somewhat.
-There is also some logging in the AWS libraries which provide some extra details.
-In particular, the setting the log `com.amazonaws.auth.AWSCredentialsProviderChain`
-to log at DEBUG level will mean the invidual reasons for the (chained)
-authentication clients to fail will be printed.
+Most common: there's an error in the key or secret.
Otherwise, try to use the AWS command line tools with the same credentials.
If you set the environment variables, you can take advantage of S3A's support
@@ -873,15 +1063,38 @@ of environment-variable authentication by attempting to use the `hdfs fs` comman
to read or write data on S3. That is: comment out the `fs.s3a` secrets and rely on
the environment variables.
+### Authentication failure when using URLs with embedded secrets
+
+If using the (strongly discouraged) mechanism of including the
+AWS Key and secret in a URL, then both "+" and "/" symbols need
+to encoded in the URL. As many AWS secrets include these characters,
+encoding problems are not uncommon.
+
+| symbol | encoded value|
+|-----------|-------------|
+| `+` | `%2B` |
+| `/` | `%2F` |
+
+
+That is, a URL for `bucket` with AWS ID `user1` and secret `a+b/c` would
+be represented as
+
+```
+s3a://user1:a%2Bb%2Fc@bucket
+```
+
+This technique is only needed when placing secrets in the URL. Again,
+this is something users are strongly advised against using.
+
### Authentication failures running on Java 8u60+
A change in the Java 8 JVM broke some of the `toString()` string generation
-of Joda Time 2.8.0, which stopped the amazon s3 client from being able to
+of Joda Time 2.8.0, which stopped the Amazon S3 client from being able to
generate authentication headers suitable for validation by S3.
Fix: make sure that the version of Joda Time is 2.8.1 or later.
-### "Bad Request" exception when working with AWS S3 Frankfurt, Seoul, or elsewhere
+### "Bad Request" exception when working with AWS S3 Frankfurt, Seoul, or other "V4" endpoint
S3 Frankfurt and Seoul *only* support
@@ -923,12 +1136,14 @@ ls: doesBucketExist on frankfurt-new: com.amazonaws.services.s3.model.AmazonS3Ex
```
This happens when trying to work with any S3 service which only supports the
-"V4" signing API \u2014and he client is configured to use the default S3A service
+"V4" signing API \u2014but the client is configured to use the default S3A service
endpoint.
The S3A client needs to be given the endpoint to use via the `fs.s3a.endpoint`
property.
+As an example, the endpoint for S3 Frankfurt is `s3.eu-central-1.amazonaws.com`:
+
```xml
<property>
<name>fs.s3a.endpoint</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index 4ff8407..5ba1871 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -18,10 +18,6 @@
package org.apache.hadoop.fs.s3a;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
-import static org.junit.Assert.*;
-
import java.io.IOException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
@@ -43,6 +39,11 @@ import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.junit.Assert.*;
+
/**
* Tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic.
*
@@ -57,6 +58,18 @@ public class TestS3AAWSCredentialsProvider {
@Rule
public ExpectedException exception = ExpectedException.none();
+ /**
+ * Declare what exception to raise, and the text which must be found
+ * in it.
+ * @param exceptionClass class of exception
+ * @param text text in exception
+ */
+ private void expectException(Class<? extends Throwable> exceptionClass,
+ String text) {
+ exception.expect(exceptionClass);
+ exception.expectMessage(text);
+ }
+
@Test
public void testBadConfiguration() throws IOException {
Configuration conf = new Configuration();
@@ -142,10 +155,35 @@ public class TestS3AAWSCredentialsProvider {
assertEquals(testFile, stat.getPath());
}
- static class ConstructorErrorProvider implements AWSCredentialsProvider {
+ /**
+ * A credential provider whose constructor signature doesn't match.
+ */
+ static class ConstructorSignatureErrorProvider
+ implements AWSCredentialsProvider {
+
+ @SuppressWarnings("unused")
+ public ConstructorSignatureErrorProvider(String str) {
+ }
+
+ @Override
+ public AWSCredentials getCredentials() {
+ return null;
+ }
+
+ @Override
+ public void refresh() {
+ }
+ }
+
+ /**
+ * A credential provider whose constructor raises an NPE.
+ */
+ static class ConstructorFailureProvider
+ implements AWSCredentialsProvider {
@SuppressWarnings("unused")
- public ConstructorErrorProvider(String str) {
+ public ConstructorFailureProvider() {
+ throw new NullPointerException("oops");
}
@Override
@@ -159,14 +197,54 @@ public class TestS3AAWSCredentialsProvider {
}
@Test
+ public void testProviderWrongClass() throws Exception {
+ expectProviderInstantiationFailure(this.getClass().getName(),
+ NOT_AWS_PROVIDER);
+ }
+
+ @Test
+ public void testProviderNotAClass() throws Exception {
+ expectProviderInstantiationFailure("NoSuchClass",
+ "ClassNotFoundException");
+ }
+
+ private void expectProviderInstantiationFailure(String option,
+ String expectedErrorText) throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER, option);
+ Path testFile = new Path(
+ conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+ expectException(IOException.class, expectedErrorText);
+ URI uri = testFile.toUri();
+ S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
+ }
+
+ @Test
public void testProviderConstructorError() throws Exception {
+ expectProviderInstantiationFailure(
+ ConstructorSignatureErrorProvider.class.getName(),
+ CONSTRUCTOR_EXCEPTION);
+ }
+
+ @Test
+ public void testProviderFailureError() throws Exception {
+ expectProviderInstantiationFailure(
+ ConstructorFailureProvider.class.getName(),
+ INSTANTIATION_EXCEPTION);
+ }
+
+ @Test
+ public void testInstantiationChain() throws Throwable {
Configuration conf = new Configuration();
conf.set(AWS_CREDENTIALS_PROVIDER,
- ConstructorErrorProvider.class.getName());
+ TemporaryAWSCredentialsProvider.NAME
+ + ", \t" + SimpleAWSCredentialsProvider.NAME
+ + " ,\n " + AnonymousAWSCredentialsProvider.NAME);
Path testFile = new Path(
conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
- exception.expect(IOException.class);
- exception.expectMessage("constructor exception");
- FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
+
+ URI uri = testFile.toUri();
+ S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
index 2d45bcc..957dfb9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
@@ -55,8 +55,8 @@ public class TestS3ATemporaryCredentials extends AbstractFSContractTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(TestS3ATemporaryCredentials.class);
- private static final String PROVIDER_CLASS =
- "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
+ private static final String PROVIDER_CLASS
+ = TemporaryAWSCredentialsProvider.NAME;
private static final long TEST_FILE_SIZE = 1024;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/763f0497/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
index c6a5a39..a22dd28 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
@@ -50,7 +50,9 @@ public class TestS3A {
@After
public void tearDown() throws Exception {
- fc.delete(getTestPath(),true);
+ if (fc != null) {
+ fc.delete(getTestPath(), true);
+ }
}
protected Path getTestPath() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-4676. Automatic and
Asynchronous Decommissioning Nodes Status Tracking. Contributed by Diniel
Zhi. (cherry picked from commit d464483bf7f0b3e3be3ba32cd6c3eee546747ab5)
Posted by cd...@apache.org.
YARN-4676. Automatic and Asynchronous Decommissioning Nodes Status Tracking. Contributed by Diniel Zhi.
(cherry picked from commit d464483bf7f0b3e3be3ba32cd6c3eee546747ab5)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0da69c32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0da69c32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0da69c32
Branch: refs/heads/HDFS-9806
Commit: 0da69c324dee9baab0f0b9700db1cc5b623f8421
Parents: 040c185
Author: Junping Du <ju...@apache.org>
Authored: Thu Aug 18 07:23:29 2016 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Thu Aug 18 07:27:23 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/util/HostsFileReader.java | 111 ++++-
.../apache/hadoop/util/TestHostsFileReader.java | 64 ++-
hadoop-project/src/site/site.xml | 1 +
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 5 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 5 +
.../hadoop/yarn/conf/YarnConfiguration.java | 14 +
.../protocolrecords/RefreshNodesRequest.java | 26 +-
..._server_resourcemanager_service_protos.proto | 1 +
.../hadoop/yarn/client/cli/RMAdminCLI.java | 166 ++++---
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 24 +-
.../impl/pb/RefreshNodesRequestPBImpl.java | 17 +-
.../src/main/resources/yarn-default.xml | 18 +
.../server/resourcemanager/AdminService.java | 3 +-
.../DecommissioningNodesWatcher.java | 439 +++++++++++++++++++
.../resourcemanager/NodesListManager.java | 166 +++++--
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../resourcemanager/ResourceTrackerService.java | 19 +
.../server/resourcemanager/rmnode/RMNode.java | 6 +
.../rmnode/RMNodeDecommissioningEvent.java | 41 ++
.../resourcemanager/rmnode/RMNodeImpl.java | 54 ++-
.../webapp/dao/ClusterMetricsInfo.java | 2 +-
.../yarn/server/resourcemanager/MockNodes.java | 5 +
.../yarn/server/resourcemanager/MockRM.java | 11 +-
.../TestDecommissioningNodesWatcher.java | 131 ++++++
.../resourcemanager/TestRMNodeTransitions.java | 11 -
.../TestResourceTrackerService.java | 199 +++++++--
.../resourcetracker/TestNMReconnect.java | 2 -
.../src/site/markdown/YarnCommands.md | 2 +-
28 files changed, 1326 insertions(+), 219 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
index 1cba426..2ef1ead 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
@@ -21,16 +21,27 @@ package org.apache.hadoop.util;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.Set;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
-import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
// Keeps track of which datanodes/tasktrackers are allowed to connect to the
// namenode/jobtracker.
@@ -38,7 +49,9 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
public class HostsFileReader {
private Set<String> includes;
- private Set<String> excludes;
+ // exclude host list with optional timeout.
+ // If the value is null, it indicates default timeout.
+ private Map<String, Integer> excludes;
private String includesFile;
private String excludesFile;
private WriteLock writeLock;
@@ -49,7 +62,7 @@ public class HostsFileReader {
public HostsFileReader(String inFile,
String exFile) throws IOException {
includes = new HashSet<String>();
- excludes = new HashSet<String>();
+ excludes = new HashMap<String, Integer>();
includesFile = inFile;
excludesFile = exFile;
ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
@@ -62,7 +75,7 @@ public class HostsFileReader {
public HostsFileReader(String includesFile, InputStream inFileInputStream,
String excludesFile, InputStream exFileInputStream) throws IOException {
includes = new HashSet<String>();
- excludes = new HashSet<String>();
+ excludes = new HashMap<String, Integer>();
this.includesFile = includesFile;
this.excludesFile = excludesFile;
ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
@@ -121,6 +134,73 @@ public class HostsFileReader {
}
}
+ public static void readFileToMap(String type,
+ String filename, Map<String, Integer> map) throws IOException {
+ File file = new File(filename);
+ FileInputStream fis = new FileInputStream(file);
+ readFileToMapWithFileInputStream(type, filename, fis, map);
+ }
+
+ public static void readFileToMapWithFileInputStream(String type,
+ String filename, InputStream inputStream, Map<String, Integer> map)
+ throws IOException {
+ // The input file could be either simple text or XML.
+ boolean xmlInput = filename.toLowerCase().endsWith(".xml");
+ if (xmlInput) {
+ readXmlFileToMapWithFileInputStream(type, filename, inputStream, map);
+ } else {
+ HashSet<String> nodes = new HashSet<String>();
+ readFileToSetWithFileInputStream(type, filename, inputStream, nodes);
+ for (String node : nodes) {
+ map.put(node, null);
+ }
+ }
+ }
+
+ public static void readXmlFileToMapWithFileInputStream(String type,
+ String filename, InputStream fileInputStream, Map<String, Integer> map)
+ throws IOException {
+ Document dom;
+ DocumentBuilderFactory builder = DocumentBuilderFactory.newInstance();
+ try {
+ DocumentBuilder db = builder.newDocumentBuilder();
+ dom = db.parse(fileInputStream);
+ // Examples:
+ // <host><name>host1</name></host>
+ // <host><name>host2</name><timeout>123</timeout></host>
+ // <host><name>host3</name><timeout>-1</timeout></host>
+ // <host><name>host4, host5,host6</name><timeout>1800</timeout></host>
+ Element doc = dom.getDocumentElement();
+ NodeList nodes = doc.getElementsByTagName("host");
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Node node = nodes.item(i);
+ if (node.getNodeType() == Node.ELEMENT_NODE) {
+ Element e= (Element) node;
+ // Support both single host and comma-separated list of hosts.
+ String v = readFirstTagValue(e, "name");
+ String[] hosts = StringUtils.getTrimmedStrings(v);
+ String str = readFirstTagValue(e, "timeout");
+ Integer timeout = (str == null)? null : Integer.parseInt(str);
+ for (String host : hosts) {
+ map.put(host, timeout);
+ LOG.info("Adding a node \"" + host + "\" to the list of "
+ + type + " hosts from " + filename);
+ }
+ }
+ }
+ } catch (IOException|SAXException|ParserConfigurationException e) {
+ LOG.fatal("error parsing " + filename, e);
+ throw new RuntimeException(e);
+ } finally {
+ fileInputStream.close();
+ }
+ }
+
+ static String readFirstTagValue(Element e, String tag) {
+ NodeList nodes = e.getElementsByTagName(tag);
+ return (nodes.getLength() == 0)? null : nodes.item(0).getTextContent();
+ }
+
public void refresh(String includeFiles, String excludeFiles)
throws IOException {
LOG.info("Refreshing hosts (include/exclude) list");
@@ -129,7 +209,7 @@ public class HostsFileReader {
// update instance variables
updateFileNames(includeFiles, excludeFiles);
Set<String> newIncludes = new HashSet<String>();
- Set<String> newExcludes = new HashSet<String>();
+ Map<String, Integer> newExcludes = new HashMap<String, Integer>();
boolean switchIncludes = false;
boolean switchExcludes = false;
if (includeFiles != null && !includeFiles.isEmpty()) {
@@ -137,7 +217,7 @@ public class HostsFileReader {
switchIncludes = true;
}
if (excludeFiles != null && !excludeFiles.isEmpty()) {
- readFileToSet("excluded", excludeFiles, newExcludes);
+ readFileToMap("excluded", excludeFiles, newExcludes);
switchExcludes = true;
}
@@ -161,7 +241,7 @@ public class HostsFileReader {
this.writeLock.lock();
try {
Set<String> newIncludes = new HashSet<String>();
- Set<String> newExcludes = new HashSet<String>();
+ Map<String, Integer> newExcludes = new HashMap<String, Integer>();
boolean switchIncludes = false;
boolean switchExcludes = false;
if (inFileInputStream != null) {
@@ -170,7 +250,7 @@ public class HostsFileReader {
switchIncludes = true;
}
if (exFileInputStream != null) {
- readFileToSetWithFileInputStream("excluded", excludesFile,
+ readFileToMapWithFileInputStream("excluded", excludesFile,
exFileInputStream, newExcludes);
switchExcludes = true;
}
@@ -199,7 +279,7 @@ public class HostsFileReader {
public Set<String> getExcludedHosts() {
this.readLock.lock();
try {
- return excludes;
+ return excludes.keySet();
} finally {
this.readLock.unlock();
}
@@ -209,7 +289,18 @@ public class HostsFileReader {
this.readLock.lock();
try {
includes.addAll(this.includes);
- excludes.addAll(this.excludes);
+ excludes.addAll(this.excludes.keySet());
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ public void getHostDetails(Set<String> includeHosts,
+ Map<String, Integer> excludeHosts) {
+ this.readLock.lock();
+ try {
+ includeHosts.addAll(this.includes);
+ excludeHosts.putAll(this.excludes);
} finally {
this.readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
index 8015f7a..5766591 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
@@ -20,16 +20,19 @@ package org.apache.hadoop.util;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
+import java.util.Map;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.*;
+
import static org.junit.Assert.*;
/*
* Test for HostsFileReader.java
- *
+ *
*/
public class TestHostsFileReader {
@@ -39,6 +42,7 @@ public class TestHostsFileReader {
File INCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.include");
String excludesFile = HOSTS_TEST_DIR + "/dfs.exclude";
String includesFile = HOSTS_TEST_DIR + "/dfs.include";
+ private String excludesXmlFile = HOSTS_TEST_DIR + "/dfs.exclude.xml";
@Before
public void setUp() throws Exception {
@@ -288,4 +292,62 @@ public class TestHostsFileReader {
assertFalse(hfp.getExcludedHosts().contains("somehost5"));
}
+
+ /*
+ * Test if timeout values are provided in HostFile
+ */
+ @Test
+ public void testHostFileReaderWithTimeout() throws Exception {
+ FileWriter efw = new FileWriter(excludesXmlFile);
+ FileWriter ifw = new FileWriter(includesFile);
+
+ efw.write("<?xml version=\"1.0\"?>\n");
+ efw.write("<!-- yarn.nodes.exclude -->\n");
+ efw.write("<hosts>\n");
+ efw.write("<host><name>host1</name></host>\n");
+ efw.write("<host><name>host2</name><timeout>123</timeout></host>\n");
+ efw.write("<host><name>host3</name><timeout>-1</timeout></host>\n");
+ efw.write("<host><name>10000</name></host>\n");
+ efw.write("<host><name>10001</name><timeout>123</timeout></host>\n");
+ efw.write("<host><name>10002</name><timeout>-1</timeout></host>\n");
+ efw.write("<host><name>host4,host5, host6</name>" +
+ "<timeout>1800</timeout></host>\n");
+ efw.write("</hosts>\n");
+ efw.close();
+
+ ifw.write("#Hosts-in-DFS\n");
+ ifw.write(" \n");
+ ifw.write(" somehost \t somehost2 \n somehost4");
+ ifw.write(" somehost3 \t # somehost5");
+ ifw.close();
+
+ HostsFileReader hfp = new HostsFileReader(includesFile, excludesXmlFile);
+
+ int includesLen = hfp.getHosts().size();
+ int excludesLen = hfp.getExcludedHosts().size();
+ assertEquals(4, includesLen);
+ assertEquals(9, excludesLen);
+
+ Set<String> includes = new HashSet<String>();
+ Map<String, Integer> excludes = new HashMap<String, Integer>();
+ hfp.getHostDetails(includes, excludes);
+ assertTrue(excludes.containsKey("host1"));
+ assertTrue(excludes.containsKey("host2"));
+ assertTrue(excludes.containsKey("host3"));
+ assertTrue(excludes.containsKey("10000"));
+ assertTrue(excludes.containsKey("10001"));
+ assertTrue(excludes.containsKey("10002"));
+ assertTrue(excludes.containsKey("host4"));
+ assertTrue(excludes.containsKey("host5"));
+ assertTrue(excludes.containsKey("host6"));
+ assertTrue(excludes.get("host1") == null);
+ assertTrue(excludes.get("host2") == 123);
+ assertTrue(excludes.get("host3") == -1);
+ assertTrue(excludes.get("10000") == null);
+ assertTrue(excludes.get("10001") == 123);
+ assertTrue(excludes.get("10002") == -1);
+ assertTrue(excludes.get("host4") == 1800);
+ assertTrue(excludes.get("host5") == 1800);
+ assertTrue(excludes.get("host6") == 1800);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 9fa1469..0d87973 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -136,6 +136,7 @@
<item name="Secure Containers" href="hadoop-yarn/hadoop-yarn-site/SecureContainer.html"/>
<item name="Registry" href="hadoop-yarn/hadoop-yarn-site/registry/index.html"/>
<item name="Reservation System" href="hadoop-yarn/hadoop-yarn-site/ReservationSystem.html"/>
+ <item name="Graceful Decommission" href="hadoop-yarn/hadoop-yarn-site/GracefulDecommission.html"/>
</menu>
<menu name="YARN REST APIs" inherit="top">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index bd737bd..c598aa0 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -213,6 +213,11 @@ public class NodeInfo {
@Override
public void setUntrackedTimeStamp(long timeStamp) {
}
+
+ @Override
+ public Integer getDecommissioningTimeout() {
+ return null;
+ }
}
public static RMNode newNodeInfo(String rackName, String hostName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 5048978..6d0ffbd 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -202,4 +202,9 @@ public class RMNodeWrapper implements RMNode {
@Override
public void setUntrackedTimeStamp(long timeStamp) {
}
+
+ @Override
+ public Integer getDecommissioningTimeout() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8899ccd..770d139 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -795,6 +795,20 @@ public class YarnConfiguration extends Configuration {
*/
public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
+ /**
+ * Timeout in seconds for YARN node graceful decommission.
+ * This is the maximal time to wait for running containers and applications
+ * to complete before transition a DECOMMISSIONING node into DECOMMISSIONED.
+ */
+ public static final String RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT =
+ RM_PREFIX + "nodemanager-graceful-decommission-timeout-secs";
+ public static final int DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT = 3600;
+
+ public static final String RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL =
+ RM_PREFIX + "decommissioning-nodes-watcher.poll-interval-secs";
+ public static final int
+ DEFAULT_RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL = 20;
+
////////////////////////////////
// Node Manager Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
index 0333c3b..732d98e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
@@ -43,6 +43,16 @@ public abstract class RefreshNodesRequest {
return request;
}
+ @Private
+ @Unstable
+ public static RefreshNodesRequest newInstance(
+ DecommissionType decommissionType, Integer timeout) {
+ RefreshNodesRequest request = Records.newRecord(RefreshNodesRequest.class);
+ request.setDecommissionType(decommissionType);
+ request.setDecommissionTimeout(timeout);
+ return request;
+ }
+
/**
* Set the DecommissionType
*
@@ -56,4 +66,18 @@ public abstract class RefreshNodesRequest {
* @return decommissionType
*/
public abstract DecommissionType getDecommissionType();
-}
+
+ /**
+ * Set the DecommissionTimeout.
+ *
+ * @param timeout graceful decommission timeout in seconds
+ */
+ public abstract void setDecommissionTimeout(Integer timeout);
+
+ /**
+ * Get the DecommissionTimeout.
+ *
+ * @return decommissionTimeout
+ */
+ public abstract Integer getDecommissionTimeout();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
index eaf658f..b9f30db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
@@ -37,6 +37,7 @@ message RefreshQueuesResponseProto {
message RefreshNodesRequestProto {
optional DecommissionTypeProto decommissionType = 1 [default = NORMAL];
+ optional int32 decommissionTimeout = 2;
}
message RefreshNodesResponseProto {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 4aa3a14..fcb9b74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -99,10 +99,10 @@ public class RMAdminCLI extends HAAdmin {
"properties. \n\t\tResourceManager will reload the " +
"mapred-queues configuration file."))
.put("-refreshNodes",
- new UsageInfo("[-g [timeout in seconds] -client|server]",
+ new UsageInfo("[-g|graceful [timeout in seconds] -client|server]",
"Refresh the hosts information at the ResourceManager. Here "
- + "[-g [timeout in seconds] -client|server] is optional, if we "
- + "specify the timeout then ResourceManager will wait for "
+ + "[-g|graceful [timeout in seconds] -client|server] is optional,"
+ + " if we specify the timeout then ResourceManager will wait for "
+ "timeout before marking the NodeManager as decommissioned."
+ " The -client|server indicates if the timeout tracking should"
+ " be handled by the client or the ResourceManager. The client"
@@ -234,21 +234,23 @@ public class RMAdminCLI extends HAAdmin {
summary.append("rmadmin is the command to execute YARN administrative " +
"commands.\n");
summary.append("The full syntax is: \n\n" +
- "yarn rmadmin" +
- " [-refreshQueues]" +
- " [-refreshNodes [-g [timeout in seconds] -client|server]]" +
- " [-refreshNodesResources]" +
- " [-refreshSuperUserGroupsConfiguration]" +
- " [-refreshUserToGroupsMappings]" +
- " [-refreshAdminAcls]" +
- " [-refreshServiceAcl]" +
- " [-getGroup [username]]" +
- " [-addToClusterNodeLabels <\"label1(exclusive=true),"
- + "label2(exclusive=false),label3\">]" +
- " [-removeFromClusterNodeLabels <label1,label2,label3>]" +
- " [-replaceLabelsOnNode <\"node1[:port]=label1,label2 node2[:port]=label1\">]" +
- " [-directlyAccessNodeLabelStore]" +
- " [-updateNodeResource [NodeID] [MemSize] [vCores] ([OvercommitTimeout])");
+ "yarn rmadmin" +
+ " [-refreshQueues]" +
+ " [-refreshNodes [-g|graceful [timeout in seconds] -client|server]]" +
+ " [-refreshNodesResources]" +
+ " [-refreshSuperUserGroupsConfiguration]" +
+ " [-refreshUserToGroupsMappings]" +
+ " [-refreshAdminAcls]" +
+ " [-refreshServiceAcl]" +
+ " [-getGroup [username]]" +
+ " [-addToClusterNodeLabels <\"label1(exclusive=true),"
+ + "label2(exclusive=false),label3\">]" +
+ " [-removeFromClusterNodeLabels <label1,label2,label3>]" +
+ " [-replaceLabelsOnNode <\"node1[:port]=label1,label2" +
+ " node2[:port]=label1\">]" +
+ " [-directlyAccessNodeLabelStore]" +
+ " [-updateNodeResource [NodeID] [MemSize] [vCores]" +
+ " ([OvercommitTimeout])");
if (isHAEnabled) {
appendHAUsage(summary);
}
@@ -309,33 +311,40 @@ public class RMAdminCLI extends HAAdmin {
return 0;
}
- private int refreshNodes() throws IOException, YarnException {
+ private int refreshNodes(boolean graceful) throws IOException, YarnException {
// Refresh the nodes
ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
- RefreshNodesRequest request = RefreshNodesRequest
- .newInstance(DecommissionType.NORMAL);
+ RefreshNodesRequest request = RefreshNodesRequest.newInstance(
+ graceful? DecommissionType.GRACEFUL : DecommissionType.NORMAL);
adminProtocol.refreshNodes(request);
return 0;
}
- private int refreshNodes(long timeout, String trackingMode)
+ private int refreshNodes(int timeout, String trackingMode)
throws IOException, YarnException {
- if (!"client".equals(trackingMode)) {
- throw new UnsupportedOperationException(
- "Only client tracking mode is currently supported.");
- }
+ boolean serverTracking = !"client".equals(trackingMode);
// Graceful decommissioning with timeout
ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
RefreshNodesRequest gracefulRequest = RefreshNodesRequest
- .newInstance(DecommissionType.GRACEFUL);
+ .newInstance(DecommissionType.GRACEFUL, timeout);
adminProtocol.refreshNodes(gracefulRequest);
+ if (serverTracking) {
+ return 0;
+ }
CheckForDecommissioningNodesRequest checkForDecommissioningNodesRequest = recordFactory
.newRecordInstance(CheckForDecommissioningNodesRequest.class);
long waitingTime;
boolean nodesDecommissioning = true;
+ // As RM enforces timeout automatically, client usually don't need
+ // to forcefully decommission nodes upon timeout.
+ // Here we let the client waits a small additional seconds so to avoid
+ // unnecessary double decommission.
+ final int gracePeriod = 5;
// timeout=-1 means wait for all the nodes to be gracefully
// decommissioned
- for (waitingTime = 0; waitingTime < timeout || timeout == -1; waitingTime++) {
+ for (waitingTime = 0;
+ timeout == -1 || (timeout >= 0 && waitingTime < timeout + gracePeriod);
+ waitingTime++) {
// wait for one second to check nodes decommissioning status
try {
Thread.sleep(1000);
@@ -380,6 +389,10 @@ public class RMAdminCLI extends HAAdmin {
return 0;
}
+ private int refreshNodes() throws IOException, YarnException {
+ return refreshNodes(false);
+ }
+
private int refreshUserToGroupsMappings() throws IOException,
YarnException {
// Refresh the user-to-groups mappings
@@ -725,33 +738,12 @@ public class RMAdminCLI extends HAAdmin {
return exitCode;
}
}
-
+
try {
if ("-refreshQueues".equals(cmd)) {
exitCode = refreshQueues();
} else if ("-refreshNodes".equals(cmd)) {
- if (args.length == 1) {
- exitCode = refreshNodes();
- } else if (args.length == 3 || args.length == 4) {
- // if the graceful timeout specified
- if ("-g".equals(args[1])) {
- long timeout = -1;
- String trackingMode;
- if (args.length == 4) {
- timeout = validateTimeout(args[2]);
- trackingMode = validateTrackingMode(args[3]);
- } else {
- trackingMode = validateTrackingMode(args[2]);
- }
- exitCode = refreshNodes(timeout, trackingMode);
- } else {
- printUsage(cmd, isHAEnabled);
- return -1;
- }
- } else {
- printUsage(cmd, isHAEnabled);
- return -1;
- }
+ exitCode = handleRefreshNodes(args, cmd, isHAEnabled);
} else if ("-refreshNodesResources".equals(cmd)) {
exitCode = refreshNodesResources();
} else if ("-refreshUserToGroupsMappings".equals(cmd)) {
@@ -768,22 +760,7 @@ public class RMAdminCLI extends HAAdmin {
String[] usernames = Arrays.copyOfRange(args, i, args.length);
exitCode = getGroups(usernames);
} else if ("-updateNodeResource".equals(cmd)) {
- if (args.length < 4 || args.length > 5) {
- System.err.println("Number of parameters specified for " +
- "updateNodeResource is wrong.");
- printUsage(cmd, isHAEnabled);
- exitCode = -1;
- } else {
- String nodeID = args[i++];
- String memSize = args[i++];
- String cores = args[i++];
- int overCommitTimeout = ResourceOption.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT;
- if (i == args.length - 1) {
- overCommitTimeout = Integer.parseInt(args[i]);
- }
- exitCode = updateNodeResource(nodeID, Integer.parseInt(memSize),
- Integer.parseInt(cores), overCommitTimeout);
- }
+ exitCode = handleUpdateNodeResource(args, cmd, isHAEnabled);
} else if ("-addToClusterNodeLabels".equals(cmd)) {
if (i >= args.length) {
System.err.println(NO_LABEL_ERR_MSG);
@@ -843,10 +820,59 @@ public class RMAdminCLI extends HAAdmin {
return exitCode;
}
- private long validateTimeout(String strTimeout) {
- long timeout;
+ // A helper method to reduce the number of lines of run()
+ private int handleRefreshNodes(String[] args, String cmd, boolean isHAEnabled)
+ throws IOException, YarnException {
+ if (args.length == 1) {
+ return refreshNodes();
+ } else if (args.length == 3 || args.length == 4) {
+ // if the graceful timeout specified
+ if ("-g".equals(args[1]) || "-graceful".equals(args[1])) {
+ int timeout = -1;
+ String trackingMode;
+ if (args.length == 4) {
+ timeout = validateTimeout(args[2]);
+ trackingMode = validateTrackingMode(args[3]);
+ } else {
+ trackingMode = validateTrackingMode(args[2]);
+ }
+ return refreshNodes(timeout, trackingMode);
+ } else {
+ printUsage(cmd, isHAEnabled);
+ return -1;
+ }
+ } else {
+ printUsage(cmd, isHAEnabled);
+ return -1;
+ }
+ }
+
+ private int handleUpdateNodeResource(
+ String[] args, String cmd, boolean isHAEnabled)
+ throws NumberFormatException, IOException, YarnException {
+ int i = 1;
+ if (args.length < 4 || args.length > 5) {
+ System.err.println("Number of parameters specified for " +
+ "updateNodeResource is wrong.");
+ printUsage(cmd, isHAEnabled);
+ return -1;
+ } else {
+ String nodeID = args[i++];
+ String memSize = args[i++];
+ String cores = args[i++];
+ int overCommitTimeout = ResourceOption.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT;
+ if (i == args.length - 1) {
+ overCommitTimeout = Integer.parseInt(args[i]);
+ }
+ return updateNodeResource(nodeID, Integer.parseInt(memSize),
+ Integer.parseInt(cores), overCommitTimeout);
+ }
+ }
+
+ private int validateTimeout(String strTimeout) {
+ int timeout;
try {
- timeout = Long.parseLong(strTimeout);
+ timeout = Integer.parseInt(strTimeout);
} catch (NumberFormatException ex) {
throw new IllegalArgumentException(INVALID_TIMEOUT_ERR_MSG + strTimeout);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
index d3161ba..60c7eac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
@@ -267,7 +267,7 @@ public class TestRMAdminCLI {
CheckForDecommissioningNodesRequest.class))).thenReturn(response);
assertEquals(0, rmAdminCLI.run(args));
verify(admin).refreshNodes(
- RefreshNodesRequest.newInstance(DecommissionType.GRACEFUL));
+ RefreshNodesRequest.newInstance(DecommissionType.GRACEFUL, 1));
verify(admin, never()).refreshNodes(
RefreshNodesRequest.newInstance(DecommissionType.FORCEFUL));
}
@@ -327,7 +327,7 @@ public class TestRMAdminCLI {
});
assertEquals(0, rmAdminCLI.run(args));
verify(admin, atLeastOnce()).refreshNodes(
- RefreshNodesRequest.newInstance(DecommissionType.GRACEFUL));
+ RefreshNodesRequest.newInstance(DecommissionType.GRACEFUL, -1));
verify(admin, never()).refreshNodes(
RefreshNodesRequest.newInstance(DecommissionType.FORCEFUL));
}
@@ -346,10 +346,6 @@ public class TestRMAdminCLI {
String[] negativeTimeoutArgs = {"-refreshNodes", "-g", "-1000", "-client"};
assertEquals(-1, rmAdminCLI.run(negativeTimeoutArgs));
- // server tracking mode
- String[] serveTrackingrArgs = {"-refreshNodes", "-g", "1", "-server"};
- assertEquals(-1, rmAdminCLI.run(serveTrackingrArgs));
-
// invalid tracking mode
String[] invalidTrackingArgs = {"-refreshNodes", "-g", "1", "-foo"};
assertEquals(-1, rmAdminCLI.run(invalidTrackingArgs));
@@ -465,8 +461,9 @@ public class TestRMAdminCLI {
assertTrue(dataOut
.toString()
.contains(
- "yarn rmadmin [-refreshQueues] [-refreshNodes [-g [timeout in " +
- "seconds] -client|server]] [-refreshNodesResources] [-refresh" +
+ "yarn rmadmin [-refreshQueues] [-refreshNodes "+
+ "[-g|graceful [timeout in seconds] -client|server]] " +
+ "[-refreshNodesResources] [-refresh" +
"SuperUserGroupsConfiguration] [-refreshUserToGroupsMappings] " +
"[-refreshAdminAcls] [-refreshServiceAcl] [-getGroup " +
"[username]] [-addToClusterNodeLabels " +
@@ -485,7 +482,8 @@ public class TestRMAdminCLI {
assertTrue(dataOut
.toString()
.contains(
- "-refreshNodes [-g [timeout in seconds] -client|server]: " +
+ "-refreshNodes [-g|graceful [timeout in seconds]" +
+ " -client|server]: " +
"Refresh the hosts information at the ResourceManager."));
assertTrue(dataOut
.toString()
@@ -518,8 +516,8 @@ public class TestRMAdminCLI {
testError(new String[] { "-help", "-refreshQueues" },
"Usage: yarn rmadmin [-refreshQueues]", dataErr, 0);
testError(new String[] { "-help", "-refreshNodes" },
- "Usage: yarn rmadmin [-refreshNodes [-g [timeout in seconds] " +
- "-client|server]]", dataErr, 0);
+ "Usage: yarn rmadmin [-refreshNodes [-g|graceful " +
+ "[timeout in seconds] -client|server]]", dataErr, 0);
testError(new String[] { "-help", "-refreshNodesResources" },
"Usage: yarn rmadmin [-refreshNodesResources]", dataErr, 0);
testError(new String[] { "-help", "-refreshUserToGroupsMappings" },
@@ -558,8 +556,8 @@ public class TestRMAdminCLI {
assertEquals(0, rmAdminCLIWithHAEnabled.run(args));
oldOutPrintStream.println(dataOut);
String expectedHelpMsg =
- "yarn rmadmin [-refreshQueues] [-refreshNodes [-g [timeout in "
- + "seconds] -client|server]] "
+ "yarn rmadmin [-refreshQueues] [-refreshNodes [-g|graceful "
+ + "[timeout in seconds] -client|server]] "
+ "[-refreshNodesResources] [-refreshSuperUserGroupsConfiguration] "
+ "[-refreshUserToGroupsMappings] "
+ "[-refreshAdminAcls] [-refreshServiceAcl] [-getGroup"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java
index 05f3230..c03a569 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java
@@ -31,7 +31,6 @@ import com.google.protobuf.TextFormat;
@Private
@Unstable
public class RefreshNodesRequestPBImpl extends RefreshNodesRequest {
-
RefreshNodesRequestProto proto = RefreshNodesRequestProto.getDefaultInstance();
RefreshNodesRequestProto.Builder builder = null;
boolean viaProto = false;
@@ -108,6 +107,22 @@ public class RefreshNodesRequestPBImpl extends RefreshNodesRequest {
return convertFromProtoFormat(p.getDecommissionType());
}
+ @Override
+ public synchronized void setDecommissionTimeout(Integer timeout) {
+ maybeInitBuilder();
+ if (timeout != null) {
+ builder.setDecommissionTimeout(timeout);
+ } else {
+ builder.clearDecommissionTimeout();
+ }
+ }
+
+ @Override
+ public synchronized Integer getDecommissionTimeout() {
+ RefreshNodesRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return p.hasDecommissionTimeout()? p.getDecommissionTimeout() : null;
+ }
+
private DecommissionType convertFromProtoFormat(DecommissionTypeProto p) {
return DecommissionType.valueOf(p.name());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e77e990..e956507 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2586,6 +2586,24 @@
</property>
<property>
+ <description>
+ Timeout in seconds for YARN node graceful decommission.
+ This is the maximal time to wait for running containers and applications to complete
+ before transition a DECOMMISSIONING node into DECOMMISSIONED.
+ </description>
+ <name>yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs</name>
+ <value>3600</value>
+ </property>
+
+ <property>
+ <description>
+ Timeout in seconds of DecommissioningNodesWatcher internal polling.
+ </description>
+ <name>yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs</name>
+ <value>20</value>
+ </property>
+
+ <property>
<description>The Node Label script to run. Script output Line starting with
"NODE_PARTITION:" will be considered as Node Label Partition. In case of
multiple lines have this pattern, then last one will be considered
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 2ec03aa..db55264 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -447,7 +447,8 @@ public class AdminService extends CompositeService implements
rmContext.getNodesListManager().refreshNodes(conf);
break;
case GRACEFUL:
- rmContext.getNodesListManager().refreshNodesGracefully(conf);
+ rmContext.getNodesListManager().refreshNodesGracefully(
+ conf, request.getDecommissionTimeout());
break;
case FORCEFUL:
rmContext.getNodesListManager().refreshNodesForcefully();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
new file mode 100644
index 0000000..376b503
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
@@ -0,0 +1,439 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+
+/**
+ * DecommissioningNodesWatcher is used by ResourceTrackerService to track
+ * DECOMMISSIONING nodes to decide when, after all running containers on
+ * the node have completed, will be transitioned into DECOMMISSIONED state
+ * (NodeManager will be told to shutdown).
+ * Under MR application, a node, after completes all its containers,
+ * may still serve it map output data during the duration of the application
+ * for reducers. A fully graceful mechanism would keep such DECOMMISSIONING
+ * nodes until all involved applications complete. It could be however
+ * undesirable under long-running applications scenario where a bunch
+ * of "idle" nodes would stay around for long period of time.
+ *
+ * DecommissioningNodesWatcher balance such concern with a timeout policy ---
+ * a DECOMMISSIONING node will be DECOMMISSIONED no later than
+ * DECOMMISSIONING_TIMEOUT regardless of running containers or applications.
+ *
+ * To be efficient, DecommissioningNodesWatcher skip tracking application
+ * containers on a particular node before the node is in DECOMMISSIONING state.
+ * It only tracks containers once the node is in DECOMMISSIONING state.
+ * DecommissioningNodesWatcher basically is no cost when no node is
+ * DECOMMISSIONING. This sacrifices the possibility that the node once
+ * host containers of an application that is still running
+ * (the affected map tasks will be rescheduled).
+ */
+public class DecommissioningNodesWatcher {
+ private static final Log LOG =
+ LogFactory.getLog(DecommissioningNodesWatcher.class);
+
+ private final RMContext rmContext;
+
+ // Default timeout value in mills.
+ // Negative value indicates no timeout. 0 means immediate.
+ private long defaultTimeoutMs =
+ 1000L * YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT;
+
+ // Once a RMNode is observed in DECOMMISSIONING state,
+ // All its ContainerStatus update are tracked inside DecomNodeContext.
+ class DecommissioningNodeContext {
+ private final NodeId nodeId;
+
+ // Last known NodeState.
+ private NodeState nodeState;
+
+ // The moment node is observed in DECOMMISSIONING state.
+ private final long decommissioningStartTime;
+
+ private long lastContainerFinishTime;
+
+ // number of running containers at the moment.
+ private int numActiveContainers;
+
+ // All applications run on the node at or after decommissioningStartTime.
+ private Set<ApplicationId> appIds;
+
+ // First moment the node is observed in DECOMMISSIONED state.
+ private long decommissionedTime;
+
+ // Timeout in millis for this decommissioning node.
+ // This value could be dynamically updated with new value from RMNode.
+ private long timeoutMs;
+
+ private long lastUpdateTime;
+
+ public DecommissioningNodeContext(NodeId nodeId) {
+ this.nodeId = nodeId;
+ this.appIds = new HashSet<ApplicationId>();
+ this.decommissioningStartTime = mclock.getTime();
+ this.timeoutMs = defaultTimeoutMs;
+ }
+
+ void updateTimeout(Integer timeoutSec) {
+ this.timeoutMs = (timeoutSec == null)?
+ defaultTimeoutMs : (1000L * timeoutSec);
+ }
+ }
+
+ // All DECOMMISSIONING nodes to track.
+ private HashMap<NodeId, DecommissioningNodeContext> decomNodes =
+ new HashMap<NodeId, DecommissioningNodeContext>();
+
+ private Timer pollTimer;
+ private MonotonicClock mclock;
+
+ public DecommissioningNodesWatcher(RMContext rmContext) {
+ this.rmContext = rmContext;
+ pollTimer = new Timer(true);
+ mclock = new MonotonicClock();
+ }
+
+ public void init(Configuration conf) {
+ readDecommissioningTimeout(conf);
+ int v = conf.getInt(
+ YarnConfiguration.RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL,
+ YarnConfiguration
+ .DEFAULT_RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL);
+ pollTimer.schedule(new PollTimerTask(rmContext), 0, (1000L * v));
+ }
+
+ /**
+ * Update rmNode decommissioning status based on NodeStatus.
+ * @param rmNode The node
+ * @param remoteNodeStatus latest NodeStatus
+ */
+ public synchronized void update(RMNode rmNode, NodeStatus remoteNodeStatus) {
+ DecommissioningNodeContext context = decomNodes.get(rmNode.getNodeID());
+ long now = mclock.getTime();
+ if (rmNode.getState() == NodeState.DECOMMISSIONED) {
+ if (context == null) {
+ return;
+ }
+ context.nodeState = rmNode.getState();
+ // keep DECOMMISSIONED node for a while for status log, so that such
+ // host will appear as DECOMMISSIONED instead of quietly disappears.
+ if (context.decommissionedTime == 0) {
+ context.decommissionedTime = now;
+ } else if (now - context.decommissionedTime > 60000L) {
+ decomNodes.remove(rmNode.getNodeID());
+ }
+ } else if (rmNode.getState() == NodeState.DECOMMISSIONING) {
+ if (context == null) {
+ context = new DecommissioningNodeContext(rmNode.getNodeID());
+ decomNodes.put(rmNode.getNodeID(), context);
+ context.nodeState = rmNode.getState();
+ context.decommissionedTime = 0;
+ }
+ context.updateTimeout(rmNode.getDecommissioningTimeout());
+ context.lastUpdateTime = now;
+
+ if (remoteNodeStatus.getKeepAliveApplications() != null) {
+ context.appIds.addAll(remoteNodeStatus.getKeepAliveApplications());
+ }
+
+ // Count number of active containers.
+ int numActiveContainers = 0;
+ for (ContainerStatus cs : remoteNodeStatus.getContainersStatuses()) {
+ ContainerState newState = cs.getState();
+ if (newState == ContainerState.RUNNING ||
+ newState == ContainerState.NEW) {
+ numActiveContainers++;
+ }
+ context.numActiveContainers = numActiveContainers;
+ ApplicationId aid = cs.getContainerId()
+ .getApplicationAttemptId().getApplicationId();
+ if (!context.appIds.contains(aid)) {
+ context.appIds.add(aid);
+ }
+ }
+
+ context.numActiveContainers = numActiveContainers;
+
+ // maintain lastContainerFinishTime.
+ if (context.numActiveContainers == 0 &&
+ context.lastContainerFinishTime == 0) {
+ context.lastContainerFinishTime = now;
+ }
+ } else {
+ // remove node in other states
+ if (context != null) {
+ decomNodes.remove(rmNode.getNodeID());
+ }
+ }
+ }
+
+ public synchronized void remove(NodeId nodeId) {
+ DecommissioningNodeContext context = decomNodes.get(nodeId);
+ if (context != null) {
+ LOG.info("remove " + nodeId + " in " + context.nodeState);
+ decomNodes.remove(nodeId);
+ }
+ }
+
+ /**
+ * Status about a specific decommissioning node.
+ *
+ */
+ public enum DecommissioningNodeStatus {
+ // Node is not in DECOMMISSIONING state.
+ NONE,
+
+ // wait for running containers to complete
+ WAIT_CONTAINER,
+
+ // wait for running application to complete (after all containers complete);
+ WAIT_APP,
+
+ // Timeout waiting for either containers or applications to complete.
+ TIMEOUT,
+
+ // nothing to wait, ready to be decommissioned
+ READY,
+
+ // The node has already been decommissioned
+ DECOMMISSIONED,
+ }
+
+ public boolean checkReadyToBeDecommissioned(NodeId nodeId) {
+ DecommissioningNodeStatus s = checkDecommissioningStatus(nodeId);
+ return (s == DecommissioningNodeStatus.READY ||
+ s == DecommissioningNodeStatus.TIMEOUT);
+ }
+
+ public DecommissioningNodeStatus checkDecommissioningStatus(NodeId nodeId) {
+ DecommissioningNodeContext context = decomNodes.get(nodeId);
+ if (context == null) {
+ return DecommissioningNodeStatus.NONE;
+ }
+
+ if (context.nodeState == NodeState.DECOMMISSIONED) {
+ return DecommissioningNodeStatus.DECOMMISSIONED;
+ }
+
+ long waitTime = mclock.getTime() - context.decommissioningStartTime;
+ if (context.numActiveContainers > 0) {
+ return (context.timeoutMs < 0 || waitTime < context.timeoutMs)?
+ DecommissioningNodeStatus.WAIT_CONTAINER :
+ DecommissioningNodeStatus.TIMEOUT;
+ }
+
+ removeCompletedApps(context);
+ if (context.appIds.size() == 0) {
+ return DecommissioningNodeStatus.READY;
+ } else {
+ return (context.timeoutMs < 0 || waitTime < context.timeoutMs)?
+ DecommissioningNodeStatus.WAIT_APP :
+ DecommissioningNodeStatus.TIMEOUT;
+ }
+ }
+
+ /**
+ * PollTimerTask periodically:
+ * 1. log status of all DECOMMISSIONING nodes;
+ * 2. identify and taken care of stale DECOMMISSIONING nodes
+ * (for example, node already terminated).
+ */
+ class PollTimerTask extends TimerTask {
+ private final RMContext rmContext;
+
+ public PollTimerTask(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ public void run() {
+ logDecommissioningNodesStatus();
+ long now = mclock.getTime();
+ Set<NodeId> staleNodes = new HashSet<NodeId>();
+
+ for (Iterator<Map.Entry<NodeId, DecommissioningNodeContext>> it =
+ decomNodes.entrySet().iterator(); it.hasNext();) {
+ Map.Entry<NodeId, DecommissioningNodeContext> e = it.next();
+ DecommissioningNodeContext d = e.getValue();
+ // Skip node recently updated (NM usually updates every second).
+ if (now - d.lastUpdateTime < 5000L) {
+ continue;
+ }
+ // Remove stale non-DECOMMISSIONING node
+ if (d.nodeState != NodeState.DECOMMISSIONING) {
+ LOG.debug("remove " + d.nodeState + " " + d.nodeId);
+ it.remove();
+ continue;
+ } else if (now - d.lastUpdateTime > 60000L) {
+ // Node DECOMMISSIONED could become stale, remove as necessary.
+ RMNode rmNode = getRmNode(d.nodeId);
+ if (rmNode != null &&
+ rmNode.getState() == NodeState.DECOMMISSIONED) {
+ LOG.debug("remove " + rmNode.getState() + " " + d.nodeId);
+ it.remove();
+ continue;
+ }
+ }
+ if (d.timeoutMs >= 0 &&
+ d.decommissioningStartTime + d.timeoutMs < now) {
+ staleNodes.add(d.nodeId);
+ LOG.debug("Identified stale and timeout node " + d.nodeId);
+ }
+ }
+
+ for (NodeId nodeId : staleNodes) {
+ RMNode rmNode = this.rmContext.getRMNodes().get(nodeId);
+ if (rmNode == null || rmNode.getState() != NodeState.DECOMMISSIONING) {
+ remove(nodeId);
+ continue;
+ }
+ if (rmNode.getState() == NodeState.DECOMMISSIONING &&
+ checkReadyToBeDecommissioned(rmNode.getNodeID())) {
+ LOG.info("DECOMMISSIONING " + nodeId + " timeout");
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION));
+ }
+ }
+ }
+ }
+
+ private RMNode getRmNode(NodeId nodeId) {
+ RMNode rmNode = this.rmContext.getRMNodes().get(nodeId);
+ if (rmNode == null) {
+ rmNode = this.rmContext.getInactiveRMNodes().get(nodeId);
+ }
+ return rmNode;
+ }
+
+ private void removeCompletedApps(DecommissioningNodeContext context) {
+ Iterator<ApplicationId> it = context.appIds.iterator();
+ while (it.hasNext()) {
+ ApplicationId appId = it.next();
+ RMApp rmApp = rmContext.getRMApps().get(appId);
+ if (rmApp == null) {
+ LOG.debug("Consider non-existing app " + appId + " as completed");
+ it.remove();
+ continue;
+ }
+ if (rmApp.getState() == RMAppState.FINISHED ||
+ rmApp.getState() == RMAppState.FAILED ||
+ rmApp.getState() == RMAppState.KILLED) {
+ LOG.debug("Remove " + rmApp.getState() + " app " + appId);
+ it.remove();
+ }
+ }
+ }
+
+ // Time in second to be decommissioned.
+ private int getTimeoutInSec(DecommissioningNodeContext context) {
+ if (context.nodeState == NodeState.DECOMMISSIONED) {
+ return 0;
+ } else if (context.nodeState != NodeState.DECOMMISSIONING) {
+ return -1;
+ }
+ if (context.appIds.size() == 0 && context.numActiveContainers == 0) {
+ return 0;
+ }
+ // negative timeout value means no timeout (infinite timeout).
+ if (context.timeoutMs < 0) {
+ return -1;
+ }
+
+ long now = mclock.getTime();
+ long timeout = context.decommissioningStartTime + context.timeoutMs - now;
+ return Math.max(0, (int)(timeout / 1000));
+ }
+
+ private void logDecommissioningNodesStatus() {
+ if (!LOG.isDebugEnabled() || decomNodes.size() == 0) {
+ return;
+ }
+ StringBuilder sb = new StringBuilder();
+ long now = mclock.getTime();
+ for (DecommissioningNodeContext d : decomNodes.values()) {
+ DecommissioningNodeStatus s = checkDecommissioningStatus(d.nodeId);
+ sb.append(String.format(
+ "%n %-34s %4ds fresh:%3ds containers:%2d %14s",
+ d.nodeId.getHost(),
+ (now - d.decommissioningStartTime) / 1000,
+ (now - d.lastUpdateTime) / 1000,
+ d.numActiveContainers,
+ s));
+ if (s == DecommissioningNodeStatus.WAIT_APP ||
+ s == DecommissioningNodeStatus.WAIT_CONTAINER) {
+ sb.append(String.format(" timeout:%4ds", getTimeoutInSec(d)));
+ }
+ for (ApplicationId aid : d.appIds) {
+ sb.append("\n " + aid);
+ RMApp rmApp = rmContext.getRMApps().get(aid);
+ if (rmApp != null) {
+ sb.append(String.format(
+ " %s %9s %5.2f%% %5ds",
+ rmApp.getState(),
+ (rmApp.getApplicationType() == null)?
+ "" : rmApp.getApplicationType(),
+ 100.0 * rmApp.getProgress(),
+ (mclock.getTime() - rmApp.getStartTime()) / 1000));
+ }
+ }
+ }
+ LOG.info("Decommissioning Nodes: " + sb.toString());
+ }
+
+ // Read possible new DECOMMISSIONING_TIMEOUT_KEY from yarn-site.xml.
+ // This enables DecommissioningNodesWatcher to pick up new value
+ // without ResourceManager restart.
+ private void readDecommissioningTimeout(Configuration conf) {
+ try {
+ if (conf == null) {
+ conf = new YarnConfiguration();
+ }
+ int v = conf.getInt(
+ YarnConfiguration.RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT,
+ YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT);
+ if (defaultTimeoutMs != 1000L * v) {
+ defaultTimeoutMs = 1000L * v;
+ LOG.info("Use new decommissioningTimeoutMs: " + defaultTimeoutMs);
+ }
+ } catch (Exception e) {
+ LOG.info("Error readDecommissioningTimeout ", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index 7937383..99413bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -19,14 +19,18 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.Map;
-import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -37,6 +41,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.HostsFileReader;
import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -47,14 +52,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeDecommissioningEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
-
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
+import com.google.common.annotations.VisibleForTesting;
+
@SuppressWarnings("unchecked")
public class NodesListManager extends CompositeService implements
EventHandler<NodesListManagerEvent> {
@@ -178,10 +184,11 @@ public class NodesListManager extends CompositeService implements
if (!LOG.isDebugEnabled()) {
return;
}
-
- LOG.debug("hostsReader: in=" + conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
+
+ LOG.debug("hostsReader: in=" +
+ conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH) + " out=" +
- conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
+ conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH));
Set<String> hostsList = new HashSet<String>();
@@ -196,23 +203,19 @@ public class NodesListManager extends CompositeService implements
}
}
- public void refreshNodes(Configuration yarnConf) throws IOException,
- YarnException {
- refreshHostsReader(yarnConf);
+ public void refreshNodes(Configuration yarnConf)
+ throws IOException, YarnException {
+ refreshNodes(yarnConf, false);
+ }
- for (NodeId nodeId: rmContext.getRMNodes().keySet()) {
- if (!isValidNode(nodeId.getHost())) {
- RMNodeEventType nodeEventType = isUntrackedNode(nodeId.getHost()) ?
- RMNodeEventType.SHUTDOWN : RMNodeEventType.DECOMMISSION;
- this.rmContext.getDispatcher().getEventHandler().handle(
- new RMNodeEvent(nodeId, nodeEventType));
- }
- }
- updateInactiveNodes();
+ public void refreshNodes(Configuration yarnConf, boolean graceful)
+ throws IOException, YarnException {
+ refreshHostsReader(yarnConf, graceful, null);
}
- private void refreshHostsReader(Configuration yarnConf) throws IOException,
- YarnException {
+ private void refreshHostsReader(
+ Configuration yarnConf, boolean graceful, Integer timeout)
+ throws IOException, YarnException {
if (null == yarnConf) {
yarnConf = new YarnConfiguration();
}
@@ -222,8 +225,16 @@ public class NodesListManager extends CompositeService implements
excludesFile =
yarnConf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH);
+ LOG.info("refreshNodes excludesFile " + excludesFile);
hostsReader.refresh(includesFile, excludesFile);
printConfiguredHosts();
+
+ LOG.info("hostsReader include:{" +
+ StringUtils.join(",", hostsReader.getHosts()) +
+ "} exclude:{" +
+ StringUtils.join(",", hostsReader.getExcludedHosts()) + "}");
+
+ handleExcludeNodeList(graceful, timeout);
}
private void setDecomissionedNMs() {
@@ -237,6 +248,86 @@ public class NodesListManager extends CompositeService implements
}
}
+ // Handle excluded nodes based on following rules:
+ // Recommission DECOMMISSIONED or DECOMMISSIONING nodes no longer excluded;
+ // Gracefully decommission excluded nodes that are not already
+ // DECOMMISSIONED nor DECOMMISSIONING; Take no action for excluded nodes
+ // that are already DECOMMISSIONED or DECOMMISSIONING.
+ private void handleExcludeNodeList(boolean graceful, Integer timeout) {
+ // DECOMMISSIONED/DECOMMISSIONING nodes need to be re-commissioned.
+ List<RMNode> nodesToRecom = new ArrayList<RMNode>();
+
+ // Nodes need to be decommissioned (graceful or forceful);
+ List<RMNode> nodesToDecom = new ArrayList<RMNode>();
+
+ Set<String> includes = new HashSet<String>();
+ Map<String, Integer> excludes = new HashMap<String, Integer>();
+ hostsReader.getHostDetails(includes, excludes);
+
+ for (RMNode n : this.rmContext.getRMNodes().values()) {
+ NodeState s = n.getState();
+ // An invalid node (either due to explicit exclude or not include)
+ // should be excluded.
+ boolean isExcluded = !isValidNode(
+ n.getHostName(), includes, excludes.keySet());
+ String nodeStr = "node " + n.getNodeID() + " with state " + s;
+ if (!isExcluded) {
+ // Note that no action is needed for DECOMMISSIONED node.
+ if (s == NodeState.DECOMMISSIONING) {
+ LOG.info("Recommission " + nodeStr);
+ nodesToRecom.add(n);
+ }
+ // Otherwise no-action needed.
+ } else {
+ // exclude is true.
+ if (graceful) {
+ // Use per node timeout if exist otherwise the request timeout.
+ Integer timeoutToUse = (excludes.get(n.getHostName()) != null)?
+ excludes.get(n.getHostName()) : timeout;
+ if (s != NodeState.DECOMMISSIONED &&
+ s != NodeState.DECOMMISSIONING) {
+ LOG.info("Gracefully decommission " + nodeStr);
+ nodesToDecom.add(n);
+ } else if (s == NodeState.DECOMMISSIONING &&
+ !Objects.equals(n.getDecommissioningTimeout(),
+ timeoutToUse)) {
+ LOG.info("Update " + nodeStr + " timeout to be " + timeoutToUse);
+ nodesToDecom.add(n);
+ } else {
+ LOG.info("No action for " + nodeStr);
+ }
+ } else {
+ if (s != NodeState.DECOMMISSIONED) {
+ LOG.info("Forcefully decommission " + nodeStr);
+ nodesToDecom.add(n);
+ }
+ }
+ }
+ }
+
+ for (RMNode n : nodesToRecom) {
+ RMNodeEvent e = new RMNodeEvent(
+ n.getNodeID(), RMNodeEventType.RECOMMISSION);
+ this.rmContext.getDispatcher().getEventHandler().handle(e);
+ }
+
+ for (RMNode n : nodesToDecom) {
+ RMNodeEvent e;
+ if (graceful) {
+ Integer timeoutToUse = (excludes.get(n.getHostName()) != null)?
+ excludes.get(n.getHostName()) : timeout;
+ e = new RMNodeDecommissioningEvent(n.getNodeID(), timeoutToUse);
+ } else {
+ RMNodeEventType eventType = isUntrackedNode(n.getHostName())?
+ RMNodeEventType.SHUTDOWN : RMNodeEventType.DECOMMISSION;
+ e = new RMNodeEvent(n.getNodeID(), eventType);
+ }
+ this.rmContext.getDispatcher().getEventHandler().handle(e);
+ }
+
+ updateInactiveNodes();
+ }
+
@VisibleForTesting
public int getNodeRemovalCheckInterval() {
return nodeRemovalCheckInterval;
@@ -360,11 +451,15 @@ public class NodesListManager extends CompositeService implements
}
public boolean isValidNode(String hostName) {
- String ip = resolver.resolve(hostName);
Set<String> hostsList = new HashSet<String>();
Set<String> excludeList = new HashSet<String>();
hostsReader.getHostDetails(hostsList, excludeList);
+ return isValidNode(hostName, hostsList, excludeList);
+ }
+ private boolean isValidNode(
+ String hostName, Set<String> hostsList, Set<String> excludeList) {
+ String ip = resolver.resolve(hostName);
return (hostsList.isEmpty() || hostsList.contains(hostName) || hostsList
.contains(ip))
&& !(excludeList.contains(hostName) || excludeList.contains(ip));
@@ -478,29 +573,14 @@ public class NodesListManager extends CompositeService implements
/**
* Refresh the nodes gracefully
*
- * @param conf
+ * @param yarnConf
+ * @param timeout decommission timeout, null means default timeout.
* @throws IOException
* @throws YarnException
*/
- public void refreshNodesGracefully(Configuration conf) throws IOException,
- YarnException {
- refreshHostsReader(conf);
- for (Entry<NodeId, RMNode> entry : rmContext.getRMNodes().entrySet()) {
- NodeId nodeId = entry.getKey();
- if (!isValidNode(nodeId.getHost())) {
- RMNodeEventType nodeEventType = isUntrackedNode(nodeId.getHost()) ?
- RMNodeEventType.SHUTDOWN : RMNodeEventType.GRACEFUL_DECOMMISSION;
- this.rmContext.getDispatcher().getEventHandler().handle(
- new RMNodeEvent(nodeId, nodeEventType));
- } else {
- // Recommissioning the nodes
- if (entry.getValue().getState() == NodeState.DECOMMISSIONING) {
- this.rmContext.getDispatcher().getEventHandler()
- .handle(new RMNodeEvent(nodeId, RMNodeEventType.RECOMMISSION));
- }
- }
- }
- updateInactiveNodes();
+ public void refreshNodesGracefully(Configuration yarnConf, Integer timeout)
+ throws IOException, YarnException {
+ refreshHostsReader(yarnConf, true, timeout);
}
/**
@@ -596,4 +676,4 @@ public class NodesListManager extends CompositeService implements
this.host = hst;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 9b9b02e..5e9827a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -70,7 +70,7 @@ public class RMServerUtils {
public static List<RMNode> queryRMNodes(RMContext context,
EnumSet<NodeState> acceptedStates) {
- // nodes contains nodes that are NEW, RUNNING OR UNHEALTHY
+ // nodes contains nodes that are NEW, RUNNING, UNHEALTHY or DECOMMISSIONING.
ArrayList<RMNode> results = new ArrayList<RMNode>();
if (acceptedStates.contains(NodeState.NEW) ||
acceptedStates.contains(NodeState.RUNNING) ||
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 9d480f3..51fc0bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -113,6 +113,8 @@ public class ResourceTrackerService extends AbstractService implements
private int minAllocMb;
private int minAllocVcores;
+ private DecommissioningNodesWatcher decommissioningWatcher;
+
private boolean isDistributedNodeLabelsConf;
private boolean isDelegatedCentralizedNodeLabelsConf;
private DynamicResourceConfiguration drConf;
@@ -131,6 +133,7 @@ public class ResourceTrackerService extends AbstractService implements
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
+ this.decommissioningWatcher = new DecommissioningNodesWatcher(rmContext);
}
@Override
@@ -170,6 +173,7 @@ public class ResourceTrackerService extends AbstractService implements
}
loadDynamicResourceConfiguration(conf);
+ decommissioningWatcher.init(conf);
super.serviceInit(conf);
}
@@ -494,6 +498,7 @@ public class ResourceTrackerService extends AbstractService implements
// Send ping
this.nmLivelinessMonitor.receivedPing(nodeId);
+ this.decommissioningWatcher.update(rmNode, remoteNodeStatus);
// 3. Check if it's a 'fresh' heartbeat i.e. not duplicate heartbeat
NodeHeartbeatResponse lastNodeHeartbeatResponse = rmNode.getLastNodeHeartBeatResponse();
@@ -526,6 +531,20 @@ public class ResourceTrackerService extends AbstractService implements
updateAppCollectorsMap(request);
}
+ // Evaluate whether a DECOMMISSIONING node is ready to be DECOMMISSIONED.
+ if (rmNode.getState() == NodeState.DECOMMISSIONING &&
+ decommissioningWatcher.checkReadyToBeDecommissioned(
+ rmNode.getNodeID())) {
+ String message = "DECOMMISSIONING " + nodeId +
+ " is ready to be decommissioned";
+ LOG.info(message);
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION));
+ this.nmLivelinessMonitor.unregister(nodeId);
+ return YarnServerBuilderUtils.newNodeHeartbeatResponse(
+ NodeAction.SHUTDOWN, message);
+ }
+
// Heartbeat response
NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils
.newNodeHeartbeatResponse(lastNodeHeartbeatResponse.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 3a9cf54..10e2afa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -175,4 +175,10 @@ public interface RMNode {
long getUntrackedTimeStamp();
void setUntrackedTimeStamp(long timeStamp);
+ /*
+ * Optional decommissioning timeout in second
+ * (null indicates default timeout).
+ * @return the decommissioning timeout in second.
+ */
+ Integer getDecommissioningTimeout();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecommissioningEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecommissioningEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecommissioningEvent.java
new file mode 100644
index 0000000..9955e9e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecommissioningEvent.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+
+/**
+ * RMNode Decommissioning Event.
+ *
+ */
+public class RMNodeDecommissioningEvent extends RMNodeEvent {
+ // Optional decommissioning timeout in second.
+ private final Integer decommissioningTimeout;
+
+ // Create instance with optional timeout
+ // (timeout could be null which means use default).
+ public RMNodeDecommissioningEvent(NodeId nodeId, Integer timeout) {
+ super(nodeId, RMNodeEventType.GRACEFUL_DECOMMISSION);
+ this.decommissioningTimeout = timeout;
+ }
+
+ public Integer getDecommissioningTimeout() {
+ return this.decommissioningTimeout;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index a3a6b30..d1ccecb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -124,6 +125,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private String healthReport;
private long lastHealthReportTime;
private String nodeManagerVersion;
+ private Integer decommissioningTimeout;
private long timeStamp;
/* Aggregated resource utilization for the containers. */
@@ -179,7 +181,6 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
NodeState,
RMNodeEventType,
RMNodeEvent>(NodeState.NEW)
-
//Transitions from NEW state
.addTransition(NodeState.NEW, NodeState.RUNNING,
RMNodeEventType.STARTED, new AddNodeTransition())
@@ -265,6 +266,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
.addTransition(NodeState.DECOMMISSIONING, NodeState.REBOOTED,
RMNodeEventType.REBOOTING,
new DeactivateNodeTransition(NodeState.REBOOTED))
+ .addTransition(NodeState.DECOMMISSIONING, NodeState.DECOMMISSIONING,
+ RMNodeEventType.FINISHED_CONTAINERS_PULLED_BY_AM,
+ new AddContainersToBeRemovedFromNMTransition())
.addTransition(NodeState.DECOMMISSIONING, NodeState.DECOMMISSIONING,
RMNodeEventType.CLEANUP_APP, new CleanUpAppTransition())
@@ -633,7 +637,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
} catch (InvalidStateTransitionException e) {
LOG.error("Can't handle this event at current state", e);
LOG.error("Invalid event " + event.getType() +
- " on Node " + this.nodeId);
+ " on Node " + this.nodeId + " oldState " + oldState);
}
if (oldState != getState()) {
LOG.info(nodeId + " Node Transitioned from " + oldState + " to "
@@ -666,6 +670,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
case SHUTDOWN:
metrics.decrNumShutdownNMs();
break;
+ case DECOMMISSIONING:
+ metrics.decrDecommissioningNMs();
+ break;
default:
LOG.debug("Unexpected previous node state");
}
@@ -712,6 +719,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
case DECOMMISSIONING:
metrics.decrDecommissioningNMs();
break;
+ case DECOMMISSIONED:
+ metrics.decrDecommisionedNMs();
+ break;
case UNHEALTHY:
metrics.decrNumUnhealthyNMs();
break;
@@ -1087,9 +1097,26 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
@Override
public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
+ Integer timeout = null;
+ if (RMNodeDecommissioningEvent.class.isInstance(event)) {
+ RMNodeDecommissioningEvent e = ((RMNodeDecommissioningEvent) event);
+ timeout = e.getDecommissioningTimeout();
+ }
+ // Pick up possible updates on decommissioningTimeout.
+ if (rmNode.getState() == NodeState.DECOMMISSIONING) {
+ if (!Objects.equals(rmNode.getDecommissioningTimeout(), timeout)) {
+ LOG.info("Update " + rmNode.getNodeID() +
+ " DecommissioningTimeout to be " + timeout);
+ rmNode.decommissioningTimeout = timeout;
+ } else {
+ LOG.info(rmNode.getNodeID() + " is already DECOMMISSIONING");
+ }
+ return;
+ }
LOG.info("Put Node " + rmNode.nodeId + " in DECOMMISSIONING.");
// Update NM metrics during graceful decommissioning.
rmNode.updateMetricsForGracefulDecommission(initState, finalState);
+ rmNode.decommissioningTimeout = timeout;
if (rmNode.originalTotalCapability == null){
rmNode.originalTotalCapability =
Resources.clone(rmNode.totalCapability);
@@ -1156,24 +1183,6 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
return NodeState.UNHEALTHY;
}
}
- if (isNodeDecommissioning) {
- List<ApplicationId> runningApps = rmNode.getRunningApps();
-
- List<ApplicationId> keepAliveApps = statusEvent.getKeepAliveAppIds();
-
- // no running (and keeping alive) app on this node, get it
- // decommissioned.
- // TODO may need to check no container is being scheduled on this node
- // as well.
- if ((runningApps == null || runningApps.size() == 0)
- && (keepAliveApps == null || keepAliveApps.size() == 0)) {
- RMNodeImpl.deactivateNode(rmNode, NodeState.DECOMMISSIONED);
- return NodeState.DECOMMISSIONED;
- }
-
- // TODO (in YARN-3223) if node in decommissioning, get node resource
- // updated if container get finished (keep available resource to be 0)
- }
rmNode.handleContainerStatus(statusEvent.getContainers());
rmNode.handleReportedIncreasedContainers(
@@ -1472,4 +1481,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
public void setUntrackedTimeStamp(long ts) {
this.timeStamp = ts;
}
+
+ @Override
+ public Integer getDecommissioningTimeout() {
+ return decommissioningTimeout;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 3012d0d..1789e09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -97,7 +97,7 @@ public class ClusterMetricsInfo {
this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
this.shutdownNodes = clusterMetrics.getNumShutdownNMs();
this.totalNodes = activeNodes + lostNodes + decommissionedNodes
- + rebootedNodes + unhealthyNodes + shutdownNodes;
+ + rebootedNodes + unhealthyNodes + decommissioningNodes + shutdownNodes;
}
public int getAppsSubmitted() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by cd...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java
new file mode 100644
index 0000000..b0b8a65
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.hadoop.util.StopWatch;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Basic unit test for S3A's blocking executor service.
+ */
+public class ITestBlockingThreadPoolExecutorService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ BlockingThreadPoolExecutorService.class);
+
+ private static final int NUM_ACTIVE_TASKS = 4;
+ private static final int NUM_WAITING_TASKS = 2;
+ private static final int TASK_SLEEP_MSEC = 100;
+ private static final int SHUTDOWN_WAIT_MSEC = 200;
+ private static final int SHUTDOWN_WAIT_TRIES = 5;
+ private static final int BLOCKING_THRESHOLD_MSEC = 50;
+
+ private static final Integer SOME_VALUE = 1337;
+
+ private static BlockingThreadPoolExecutorService tpe = null;
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ ensureDestroyed();
+ }
+
+ /**
+ * Basic test of running one trivial task.
+ */
+ @Test
+ public void testSubmitCallable() throws Exception {
+ ensureCreated();
+ ListenableFuture<Integer> f = tpe.submit(callableSleeper);
+ Integer v = f.get();
+ assertEquals(SOME_VALUE, v);
+ }
+
+ /**
+ * More involved test, including detecting blocking when at capacity.
+ */
+ @Test
+ public void testSubmitRunnable() throws Exception {
+ ensureCreated();
+ int totalTasks = NUM_ACTIVE_TASKS + NUM_WAITING_TASKS;
+ StopWatch stopWatch = new StopWatch().start();
+ for (int i = 0; i < totalTasks; i++) {
+ tpe.submit(sleeper);
+ assertDidntBlock(stopWatch);
+ }
+ tpe.submit(sleeper);
+ assertDidBlock(stopWatch);
+ }
+
+ @Test
+ public void testShutdown() throws Exception {
+ // Cover create / destroy, regardless of when this test case runs
+ ensureCreated();
+ ensureDestroyed();
+
+ // Cover create, execute, destroy, regardless of when test case runs
+ ensureCreated();
+ testSubmitRunnable();
+ ensureDestroyed();
+ }
+
+ // Helper functions, etc.
+
+ private void assertDidntBlock(StopWatch sw) {
+ try {
+ assertFalse("Non-blocking call took too long.",
+ sw.now(TimeUnit.MILLISECONDS) > BLOCKING_THRESHOLD_MSEC);
+ } finally {
+ sw.reset().start();
+ }
+ }
+
+ private void assertDidBlock(StopWatch sw) {
+ try {
+ if (sw.now(TimeUnit.MILLISECONDS) < BLOCKING_THRESHOLD_MSEC) {
+ throw new RuntimeException("Blocking call returned too fast.");
+ }
+ } finally {
+ sw.reset().start();
+ }
+ }
+
+ private Runnable sleeper = new Runnable() {
+ @Override
+ public void run() {
+ String name = Thread.currentThread().getName();
+ try {
+ Thread.sleep(TASK_SLEEP_MSEC);
+ } catch (InterruptedException e) {
+ LOG.info("Thread {} interrupted.", name);
+ Thread.currentThread().interrupt();
+ }
+ }
+ };
+
+ private Callable<Integer> callableSleeper = new Callable<Integer>() {
+ @Override
+ public Integer call() throws Exception {
+ sleeper.run();
+ return SOME_VALUE;
+ }
+ };
+
+ /**
+ * Helper function to create thread pool under test.
+ */
+ private static void ensureCreated() throws Exception {
+ if (tpe == null) {
+ LOG.debug("Creating thread pool");
+ tpe = new BlockingThreadPoolExecutorService(NUM_ACTIVE_TASKS,
+ NUM_WAITING_TASKS, 1, TimeUnit.SECONDS, "btpetest");
+ }
+ }
+
+ /**
+ * Helper function to terminate thread pool under test, asserting that
+ * shutdown -> terminate works as expected.
+ */
+ private static void ensureDestroyed() throws Exception {
+ if (tpe == null) {
+ return;
+ }
+ int shutdownTries = SHUTDOWN_WAIT_TRIES;
+
+ tpe.shutdown();
+ if (!tpe.isShutdown()) {
+ throw new RuntimeException("Shutdown had no effect.");
+ }
+
+ while (!tpe.awaitTermination(SHUTDOWN_WAIT_MSEC,
+ TimeUnit.MILLISECONDS)) {
+ LOG.info("Waiting for thread pool shutdown.");
+ if (shutdownTries-- <= 0) {
+ LOG.error("Failed to terminate thread pool gracefully.");
+ break;
+ }
+ }
+ if (!tpe.isTerminated()) {
+ tpe.shutdownNow();
+ if (!tpe.awaitTermination(SHUTDOWN_WAIT_MSEC,
+ TimeUnit.MILLISECONDS)) {
+ throw new RuntimeException(
+ "Failed to terminate thread pool in timely manner.");
+ }
+ }
+ tpe = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
new file mode 100644
index 0000000..cf8783c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.AccessDeniedException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSCredentialsProviderChain;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.junit.Assert.*;
+
+/**
+ * Tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic.
+ *
+ */
+public class ITestS3AAWSCredentialsProvider {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AAWSCredentialsProvider.class);
+
+ @Rule
+ public Timeout testTimeout = new Timeout(1 * 60 * 1000);
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ /**
+ * Declare what exception to raise, and the text which must be found
+ * in it.
+ * @param exceptionClass class of exception
+ * @param text text in exception
+ */
+ private void expectException(Class<? extends Throwable> exceptionClass,
+ String text) {
+ exception.expect(exceptionClass);
+ exception.expectMessage(text);
+ }
+
+ @Test
+ public void testBadConfiguration() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER, "no.such.class");
+ try {
+ createFailingFS(conf);
+ } catch (IOException e) {
+ if (!(e.getCause() instanceof ClassNotFoundException)) {
+ LOG.error("Unexpected nested cause: {} in {}", e.getCause(), e, e);
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Create a filesystem, expect it to fail by raising an IOException.
+ * Raises an assertion exception if in fact the FS does get instantiated.
+ * @param conf configuration
+ * @throws IOException an expected exception.
+ */
+ private void createFailingFS(Configuration conf) throws IOException {
+ S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf);
+ fs.listStatus(new Path("/"));
+ fail("Expected exception - got " + fs);
+ }
+
+ static class BadCredentialsProvider implements AWSCredentialsProvider {
+
+ @SuppressWarnings("unused")
+ public BadCredentialsProvider(URI name, Configuration conf) {
+ }
+
+ @Override
+ public AWSCredentials getCredentials() {
+ return new BasicAWSCredentials("bad_key", "bad_secret");
+ }
+
+ @Override
+ public void refresh() {
+ }
+ }
+
+ @Test
+ public void testBadCredentials() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER, BadCredentialsProvider.class.getName());
+ try {
+ createFailingFS(conf);
+ } catch (AccessDeniedException e) {
+ // expected
+ }
+ }
+
+ static class GoodCredentialsProvider extends AWSCredentialsProviderChain {
+
+ @SuppressWarnings("unused")
+ public GoodCredentialsProvider(URI name, Configuration conf) {
+ super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
+ conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider());
+ }
+ }
+
+ @Test
+ public void testGoodProvider() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName());
+ S3ATestUtils.createTestFileSystem(conf);
+ }
+
+ @Test
+ public void testAnonymousProvider() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER,
+ AnonymousAWSCredentialsProvider.class.getName());
+ Path testFile = new Path(
+ conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+ S3ATestUtils.useCSVDataEndpoint(conf);
+ FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
+ assertNotNull(fs);
+ assertTrue(fs instanceof S3AFileSystem);
+ FileStatus stat = fs.getFileStatus(testFile);
+ assertNotNull(stat);
+ assertEquals(testFile, stat.getPath());
+ }
+
+ /**
+ * A credential provider whose constructor signature doesn't match.
+ */
+ static class ConstructorSignatureErrorProvider
+ implements AWSCredentialsProvider {
+
+ @SuppressWarnings("unused")
+ public ConstructorSignatureErrorProvider(String str) {
+ }
+
+ @Override
+ public AWSCredentials getCredentials() {
+ return null;
+ }
+
+ @Override
+ public void refresh() {
+ }
+ }
+
+ /**
+ * A credential provider whose constructor raises an NPE.
+ */
+ static class ConstructorFailureProvider
+ implements AWSCredentialsProvider {
+
+ @SuppressWarnings("unused")
+ public ConstructorFailureProvider() {
+ throw new NullPointerException("oops");
+ }
+
+ @Override
+ public AWSCredentials getCredentials() {
+ return null;
+ }
+
+ @Override
+ public void refresh() {
+ }
+ }
+
+ @Test
+ public void testProviderWrongClass() throws Exception {
+ expectProviderInstantiationFailure(this.getClass().getName(),
+ NOT_AWS_PROVIDER);
+ }
+
+ @Test
+ public void testProviderNotAClass() throws Exception {
+ expectProviderInstantiationFailure("NoSuchClass",
+ "ClassNotFoundException");
+ }
+
+ private void expectProviderInstantiationFailure(String option,
+ String expectedErrorText) throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER, option);
+ Path testFile = new Path(
+ conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+ expectException(IOException.class, expectedErrorText);
+ URI uri = testFile.toUri();
+ S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
+ }
+
+ @Test
+ public void testProviderConstructorError() throws Exception {
+ expectProviderInstantiationFailure(
+ ConstructorSignatureErrorProvider.class.getName(),
+ CONSTRUCTOR_EXCEPTION);
+ }
+
+ @Test
+ public void testProviderFailureError() throws Exception {
+ expectProviderInstantiationFailure(
+ ConstructorFailureProvider.class.getName(),
+ INSTANTIATION_EXCEPTION);
+ }
+
+ @Test
+ public void testInstantiationChain() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER,
+ TemporaryAWSCredentialsProvider.NAME
+ + ", \t" + SimpleAWSCredentialsProvider.NAME
+ + " ,\n " + AnonymousAWSCredentialsProvider.NAME);
+ Path testFile = new Path(
+ conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+
+ URI uri = testFile.toUri();
+ S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java
new file mode 100644
index 0000000..4444d0c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+/**
+ * Demonstrate that the threadpool blocks additional client requests if
+ * its queue is full (rather than throwing an exception) by initiating an
+ * upload consisting of 4 parts with 2 threads and 1 spot in the queue. The
+ * 4th part should not trigger an exception as it would with a
+ * non-blocking threadpool.
+ */
+public class ITestS3ABlockingThreadPool {
+
+ private Configuration conf;
+ private S3AFileSystem fs;
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ protected Path getTestPath() {
+ return new Path("/tests3a");
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024);
+ conf.setLong(Constants.MULTIPART_SIZE, 5 * 1024 * 1024);
+ conf.setInt(Constants.MAX_THREADS, 2);
+ conf.setInt(Constants.MAX_TOTAL_TASKS, 1);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(getTestPath(), true);
+ }
+ }
+
+ @Test
+ public void testRegularMultiPartUpload() throws Exception {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 *
+ 1024);
+ }
+
+ @Test
+ public void testFastMultiPartUpload() throws Exception {
+ conf.setBoolean(Constants.FAST_UPLOAD, true);
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 *
+ 1024);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java
new file mode 100644
index 0000000..9a6dae7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.fileStatsToString;
+
+/**
+ * S3A tests for configuring block size.
+ */
+public class ITestS3ABlocksize extends AbstractFSContractTestBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3ABlocksize.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testBlockSize() throws Exception {
+ FileSystem fs = getFileSystem();
+ long defaultBlockSize = fs.getDefaultBlockSize();
+ assertEquals("incorrect blocksize",
+ S3AFileSystem.DEFAULT_BLOCKSIZE, defaultBlockSize);
+ long newBlockSize = defaultBlockSize * 2;
+ fs.getConf().setLong(Constants.FS_S3A_BLOCK_SIZE, newBlockSize);
+
+ Path dir = path("testBlockSize");
+ Path file = new Path(dir, "file");
+ createFile(fs, file, true, dataset(1024, 'a', 'z' - 'a'));
+ FileStatus fileStatus = fs.getFileStatus(file);
+ assertEquals("Double default block size in stat(): " + fileStatus,
+ newBlockSize,
+ fileStatus.getBlockSize());
+
+ // check the listing & assert that the block size is picked up by
+ // this route too.
+ boolean found = false;
+ FileStatus[] listing = fs.listStatus(dir);
+ for (FileStatus stat : listing) {
+ LOG.info("entry: {}", stat);
+ if (file.equals(stat.getPath())) {
+ found = true;
+ assertEquals("Double default block size in ls(): " + stat,
+ newBlockSize,
+ stat.getBlockSize());
+ }
+ }
+ assertTrue("Did not find " + fileStatsToString(listing, ", "), found);
+ }
+
+ @Test
+ public void testRootFileStatusHasBlocksize() throws Throwable {
+ FileSystem fs = getFileSystem();
+ FileStatus status = fs.getFileStatus(new Path("/"));
+ assertTrue("Invalid root blocksize",
+ status.getBlockSize() >= 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
new file mode 100644
index 0000000..4e99339
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -0,0 +1,435 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.reflect.FieldUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3native.S3xLoginHelper;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.net.URI;
+
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.http.HttpStatus;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * S3A tests for configuration.
+ */
+public class ITestS3AConfiguration {
+ private static final String EXAMPLE_ID = "AKASOMEACCESSKEY";
+ private static final String EXAMPLE_KEY =
+ "RGV0cm9pdCBSZ/WQgY2xl/YW5lZCB1cAEXAMPLE";
+
+ private Configuration conf;
+ private S3AFileSystem fs;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AConfiguration.class);
+
+ private static final String TEST_ENDPOINT = "test.fs.s3a.endpoint";
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Rule
+ public final TemporaryFolder tempDir = new TemporaryFolder();
+
+ /**
+ * Test if custom endpoint is picked up.
+ * <p/>
+ * The test expects TEST_ENDPOINT to be defined in the Configuration
+ * describing the endpoint of the bucket to which TEST_FS_S3A_NAME points
+ * (f.i. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland).
+ * Evidently, the bucket has to be hosted in the region denoted by the
+ * endpoint for the test to succeed.
+ * <p/>
+ * More info and the list of endpoint identifiers:
+ * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testEndpoint() throws Exception {
+ conf = new Configuration();
+ String endpoint = conf.getTrimmed(TEST_ENDPOINT, "");
+ if (endpoint.isEmpty()) {
+ LOG.warn("Custom endpoint test skipped as " + TEST_ENDPOINT + "config " +
+ "setting was not detected");
+ } else {
+ conf.set(Constants.ENDPOINT, endpoint);
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ AmazonS3Client s3 = fs.getAmazonS3Client();
+ String endPointRegion = "";
+ // Differentiate handling of "s3-" and "s3." based endpoint identifiers
+ String[] endpointParts = StringUtils.split(endpoint, '.');
+ if (endpointParts.length == 3) {
+ endPointRegion = endpointParts[0].substring(3);
+ } else if (endpointParts.length == 4) {
+ endPointRegion = endpointParts[1];
+ } else {
+ fail("Unexpected endpoint");
+ }
+ assertEquals("Endpoint config setting and bucket location differ: ",
+ endPointRegion, s3.getBucketLocation(fs.getUri().getHost()));
+ }
+ }
+
+ @Test
+ public void testProxyConnection() throws Exception {
+ conf = new Configuration();
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.set(Constants.PROXY_HOST, "127.0.0.1");
+ conf.setInt(Constants.PROXY_PORT, 1);
+ String proxy =
+ conf.get(Constants.PROXY_HOST) + ":" + conf.get(Constants.PROXY_PORT);
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server at " + proxy);
+ } catch (AWSClientIOException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testProxyPortWithoutHost() throws Exception {
+ conf = new Configuration();
+ conf.unset(Constants.PROXY_HOST);
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.setInt(Constants.PROXY_PORT, 1);
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a proxy configuration error");
+ } catch (IllegalArgumentException e) {
+ String msg = e.toString();
+ if (!msg.contains(Constants.PROXY_HOST) &&
+ !msg.contains(Constants.PROXY_PORT)) {
+ throw e;
+ }
+ }
+ }
+
+ @Test
+ public void testAutomaticProxyPortSelection() throws Exception {
+ conf = new Configuration();
+ conf.unset(Constants.PROXY_PORT);
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.set(Constants.PROXY_HOST, "127.0.0.1");
+ conf.set(Constants.SECURE_CONNECTIONS, "true");
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server");
+ } catch (AWSClientIOException e) {
+ // expected
+ }
+ conf.set(Constants.SECURE_CONNECTIONS, "false");
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server");
+ } catch (AWSClientIOException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testUsernameInconsistentWithPassword() throws Exception {
+ conf = new Configuration();
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.set(Constants.PROXY_HOST, "127.0.0.1");
+ conf.setInt(Constants.PROXY_PORT, 1);
+ conf.set(Constants.PROXY_USERNAME, "user");
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server");
+ } catch (IllegalArgumentException e) {
+ String msg = e.toString();
+ if (!msg.contains(Constants.PROXY_USERNAME) &&
+ !msg.contains(Constants.PROXY_PASSWORD)) {
+ throw e;
+ }
+ }
+ conf = new Configuration();
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.set(Constants.PROXY_HOST, "127.0.0.1");
+ conf.setInt(Constants.PROXY_PORT, 1);
+ conf.set(Constants.PROXY_PASSWORD, "password");
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server");
+ } catch (IllegalArgumentException e) {
+ String msg = e.toString();
+ if (!msg.contains(Constants.PROXY_USERNAME) &&
+ !msg.contains(Constants.PROXY_PASSWORD)) {
+ throw e;
+ }
+ }
+ }
+
+ @Test
+ public void testCredsFromCredentialProvider() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ provisionAccessKeys(conf);
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
+ assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
+ }
+
+ void provisionAccessKeys(final Configuration conf) throws Exception {
+ // add our creds to the provider
+ final CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(Constants.ACCESS_KEY,
+ EXAMPLE_ID.toCharArray());
+ provider.createCredentialEntry(Constants.SECRET_KEY,
+ EXAMPLE_KEY.toCharArray());
+ provider.flush();
+ }
+
+ @Test
+ public void testCredsFromUserInfo() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ provisionAccessKeys(conf);
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ URI uriWithUserInfo = new URI("s3a://123:456@foobar");
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
+ assertEquals("AccessKey incorrect.", "123", creds.getUser());
+ assertEquals("SecretKey incorrect.", "456", creds.getPassword());
+ }
+
+ @Test
+ public void testIDFromUserInfoSecretFromCredentialProvider()
+ throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ provisionAccessKeys(conf);
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ URI uriWithUserInfo = new URI("s3a://123@foobar");
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
+ assertEquals("AccessKey incorrect.", "123", creds.getUser());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
+ }
+
+ @Test
+ public void testSecretFromCredentialProviderIDFromConfig() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ // add our creds to the provider
+ final CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(Constants.SECRET_KEY,
+ EXAMPLE_KEY.toCharArray());
+ provider.flush();
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID);
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
+ assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
+ }
+
+ @Test
+ public void testIDFromCredentialProviderSecretFromConfig() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ // add our creds to the provider
+ final CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(Constants.ACCESS_KEY,
+ EXAMPLE_ID.toCharArray());
+ provider.flush();
+
+ conf.set(Constants.SECRET_KEY, EXAMPLE_KEY);
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
+ assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
+ }
+
+ @Test
+ public void testExcludingS3ACredentialProvider() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ "jceks://s3a/foobar," + jks.toString());
+
+ // first make sure that the s3a based provider is removed
+ Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, S3AFileSystem.class);
+ String newPath = conf.get(
+ CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
+ assertFalse("Provider Path incorrect", newPath.contains("s3a://"));
+
+ // now let's make sure the new path is created by the S3AFileSystem
+ // and the integration still works. Let's provision the keys through
+ // the altered configuration instance and then try and access them
+ // using the original config with the s3a provider in the path.
+ provisionAccessKeys(c);
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ URI uriWithUserInfo = new URI("s3a://123:456@foobar");
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
+ assertEquals("AccessKey incorrect.", "123", creds.getUser());
+ assertEquals("SecretKey incorrect.", "456", creds.getPassword());
+
+ }
+
+ @Test
+ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty()
+ throws Exception {
+
+ conf = new Configuration();
+ conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true));
+ assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false));
+
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ assertNotNull(fs);
+ AmazonS3Client s3 = fs.getAmazonS3Client();
+ assertNotNull(s3);
+ S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class,
+ "clientOptions");
+ assertTrue("Expected to find path style access to be switched on!",
+ clientOptions.isPathStyleAccess());
+ byte[] file = ContractTestUtils.toAsciiByteArray("test file");
+ ContractTestUtils.writeAndRead(fs,
+ new Path("/path/style/access/testFile"), file, file.length,
+ conf.getInt(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true);
+ } catch (final AWSS3IOException e) {
+ LOG.error("Caught exception: ", e);
+ // Catch/pass standard path style access behaviour when live bucket
+ // isn't in the same region as the s3 client default. See
+ // http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
+ assertEquals(e.getStatusCode(), HttpStatus.SC_MOVED_PERMANENTLY);
+ }
+ }
+
+ @Test
+ public void testDefaultUserAgent() throws Exception {
+ conf = new Configuration();
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ assertNotNull(fs);
+ AmazonS3Client s3 = fs.getAmazonS3Client();
+ assertNotNull(s3);
+ ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
+ "clientConfiguration");
+ assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent());
+ }
+
+ @Test
+ public void testCustomUserAgent() throws Exception {
+ conf = new Configuration();
+ conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ assertNotNull(fs);
+ AmazonS3Client s3 = fs.getAmazonS3Client();
+ assertNotNull(s3);
+ ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
+ "clientConfiguration");
+ assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
+ awsConf.getUserAgent());
+ }
+
+ /**
+ * Reads and returns a field from an object using reflection. If the field
+ * cannot be found, is null, or is not the expected type, then this method
+ * fails the test.
+ *
+ * @param target object to read
+ * @param fieldType type of field to read, which will also be the return type
+ * @param fieldName name of field to read
+ * @return field that was read
+ * @throws IllegalAccessException if access not allowed
+ */
+ private static <T> T getField(Object target, Class<T> fieldType,
+ String fieldName) throws IllegalAccessException {
+ Object obj = FieldUtils.readField(target, fieldName, true);
+ assertNotNull(String.format(
+ "Could not read field named %s in object with class %s.", fieldName,
+ target.getClass().getName()), obj);
+ assertTrue(String.format(
+ "Unexpected type found for field named %s, expected %s, actual %s.",
+ fieldName, fieldType.getName(), obj.getClass().getName()),
+ fieldType.isAssignableFrom(obj.getClass()));
+ return fieldType.cast(obj);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java
new file mode 100644
index 0000000..b3d7abf
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.file.AccessDeniedException;
+
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.TEST_FS_S3A_NAME;
+
+/**
+ * Tests that credentials can go into the URL. This includes a valid
+ * set, and a check that an invalid set do at least get stripped out
+ * of the final URI
+ */
+public class ITestS3ACredentialsInURL extends Assert {
+ private S3AFileSystem fs;
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3ACredentialsInURL.class);
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @After
+ public void teardown() {
+ IOUtils.closeStream(fs);
+ }
+
+ /**
+ * Test instantiation.
+ * @throws Throwable
+ */
+ @Test
+ public void testInstantiateFromURL() throws Throwable {
+
+ Configuration conf = new Configuration();
+ String accessKey = conf.get(Constants.ACCESS_KEY);
+ String secretKey = conf.get(Constants.SECRET_KEY);
+ String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, "");
+ Assume.assumeNotNull(fsname, accessKey, secretKey);
+ URI original = new URI(fsname);
+ URI secretsURI = createUriWithEmbeddedSecrets(original,
+ accessKey, secretKey);
+ if (secretKey.contains("/")) {
+ assertTrue("test URI encodes the / symbol", secretsURI.toString().
+ contains("%252F"));
+ }
+ if (secretKey.contains("+")) {
+ assertTrue("test URI encodes the + symbol", secretsURI.toString().
+ contains("%252B"));
+ }
+ assertFalse("Does not contain secrets", original.equals(secretsURI));
+
+ conf.set(TEST_FS_S3A_NAME, secretsURI.toString());
+ conf.unset(Constants.ACCESS_KEY);
+ conf.unset(Constants.SECRET_KEY);
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ String fsURI = fs.getUri().toString();
+ assertFalse("FS URI contains a @ symbol", fsURI.contains("@"));
+ assertFalse("FS URI contains a % symbol", fsURI.contains("%"));
+ if (!original.toString().startsWith(fsURI)) {
+ fail("Filesystem URI does not match original");
+ }
+ validate("original path", new Path(original));
+ validate("bare path", new Path("/"));
+ validate("secrets path", new Path(secretsURI));
+ }
+
+ private void validate(String text, Path path) throws IOException {
+ try {
+ fs.canonicalizeUri(path.toUri());
+ fs.checkPath(path);
+ assertTrue(text + " Not a directory",
+ fs.getFileStatus(new Path("/")).isDirectory());
+ fs.globStatus(path);
+ } catch (AssertionError e) {
+ throw e;
+ } catch (Exception e) {
+ LOG.debug("{} failure: {}", text, e, e);
+ fail(text + " Test failed");
+ }
+ }
+
+ /**
+ * Set up some invalid credentials, verify login is rejected.
+ * @throws Throwable
+ */
+ @Test
+ public void testInvalidCredentialsFail() throws Throwable {
+ Configuration conf = new Configuration();
+ String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, "");
+ Assume.assumeNotNull(fsname);
+ URI original = new URI(fsname);
+ URI testURI = createUriWithEmbeddedSecrets(original, "user", "//");
+
+ conf.set(TEST_FS_S3A_NAME, testURI.toString());
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ try {
+ S3AFileStatus status = fs.getFileStatus(new Path("/"));
+ fail("Expected an AccessDeniedException, got " + status);
+ } catch (AccessDeniedException e) {
+ // expected
+ }
+
+ }
+
+ private URI createUriWithEmbeddedSecrets(URI original,
+ String accessKey,
+ String secretKey) throws UnsupportedEncodingException {
+ String encodedSecretKey = URLEncoder.encode(secretKey, "UTF-8");
+ String formattedString = String.format("%s://%s:%s@%s/%s/",
+ original.getScheme(),
+ accessKey,
+ encodedSecretKey,
+ original.getHost(),
+ original.getPath());
+ URI testURI;
+ try {
+ testURI = new Path(formattedString).toUri();
+ } catch (IllegalArgumentException e) {
+ // inner cause is stripped to keep any secrets out of stack traces
+ throw new IllegalArgumentException("Could not encode Path");
+ }
+ return testURI;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java
new file mode 100644
index 0000000..4543278
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+
+/**
+ * Test whether or not encryption works by turning it on. Some checks
+ * are made for different file sizes as there have been reports that the
+ * file length may be rounded up to match word boundaries.
+ */
+public class ITestS3AEncryption extends AbstractS3ATestBase {
+ private static final String AES256 = Constants.SERVER_SIDE_ENCRYPTION_AES256;
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ AES256);
+ return conf;
+ }
+
+ private static final int[] SIZES = {
+ 0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 10 - 3, 2 ^ 11 - 2, 2 ^ 12 - 1
+ };
+
+ @Override
+ public void teardown() throws Exception {
+ super.teardown();
+ IOUtils.closeStream(getFileSystem());
+ }
+
+ @Test
+ public void testEncryption() throws Throwable {
+ for (int size: SIZES) {
+ validateEncryptionForFilesize(size);
+ }
+ }
+
+ @Test
+ public void testEncryptionOverRename() throws Throwable {
+ skipIfEncryptionTestsDisabled(getConfiguration());
+ Path src = path(createFilename(1024));
+ byte[] data = dataset(1024, 'a', 'z');
+ S3AFileSystem fs = getFileSystem();
+ writeDataset(fs, src, data, data.length, 1024 * 1024, true);
+ ContractTestUtils.verifyFileContents(fs, src, data);
+ Path dest = path(src.getName() + "-copy");
+ fs.rename(src, dest);
+ ContractTestUtils.verifyFileContents(fs, dest, data);
+ assertEncrypted(dest);
+ }
+
+ protected void validateEncryptionForFilesize(int len) throws IOException {
+ skipIfEncryptionTestsDisabled(getConfiguration());
+ describe("Create an encrypted file of size " + len);
+ String src = createFilename(len);
+ Path path = writeThenReadFile(src, len);
+ assertEncrypted(path);
+ rm(getFileSystem(), path, false, false);
+ }
+
+ private String createFilename(int len) {
+ return String.format("%s-%04x", methodName.getMethodName(), len);
+ }
+
+ /**
+ * Assert that at path references an encrypted blob.
+ * @param path path
+ * @throws IOException on a failure
+ */
+ private void assertEncrypted(Path path) throws IOException {
+ ObjectMetadata md = getFileSystem().getObjectMetadata(path);
+ assertEquals(AES256, md.getSSEAlgorithm());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java
new file mode 100644
index 0000000..81578c2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+
+/**
+ * Test whether or not encryption settings propagate by choosing an invalid
+ * one. We expect the write to fail with a 400 bad request error
+ */
+public class ITestS3AEncryptionAlgorithmPropagation
+ extends AbstractS3ATestBase {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ "DES");
+ return conf;
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ super.teardown();
+ IOUtils.closeStream(getFileSystem());
+ }
+
+ @Test
+ public void testEncrypt0() throws Throwable {
+ writeThenReadFileToFailure(0);
+ }
+
+ @Test
+ public void testEncrypt256() throws Throwable {
+ writeThenReadFileToFailure(256);
+ }
+
+ /**
+ * Make this a no-op so test setup doesn't fail.
+ * @param path path path
+ * @throws IOException on any failure
+ */
+ @Override
+ protected void mkdirs(Path path) throws IOException {
+
+ }
+
+ protected void writeThenReadFileToFailure(int len) throws IOException {
+ skipIfEncryptionTestsDisabled(getConfiguration());
+ describe("Create an encrypted file of size " + len);
+ try {
+ writeThenReadFile(methodName.getMethodName() + '-' + len, len);
+ fail("Expected an exception about an illegal encryption algorithm");
+ } catch (AWSS3IOException e) {
+ assertStatusCode(e, 400);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionFastOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionFastOutputStream.java
new file mode 100644
index 0000000..c06fed1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionFastOutputStream.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Run the encryption tests against the Fast output stream.
+ * This verifies that both file writing paths can encrypt their data.
+ */
+public class ITestS3AEncryptionFastOutputStream extends ITestS3AEncryption {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.setBoolean(Constants.FAST_UPLOAD, true);
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
new file mode 100644
index 0000000..d8e017e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.nio.file.AccessDeniedException;
+import java.util.concurrent.Callable;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+
+/**
+ * Test S3A Failure translation, including a functional test
+ * generating errors during stream IO.
+ */
+public class ITestS3AFailureHandling extends AbstractFSContractTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AFailureHandling.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Test
+ public void testReadFileChanged() throws Throwable {
+ describe("overwrite a file with a shorter one during a read, seek");
+ final int fullLength = 8192;
+ final byte[] fullDataset = dataset(fullLength, 'a', 32);
+ final int shortLen = 4096;
+ final byte[] shortDataset = dataset(shortLen, 'A', 32);
+ final FileSystem fs = getFileSystem();
+ final Path testpath = path("readFileToChange.txt");
+ // initial write
+ writeDataset(fs, testpath, fullDataset, fullDataset.length, 1024, false);
+ try(FSDataInputStream instream = fs.open(testpath)) {
+ instream.seek(fullLength - 16);
+ assertTrue("no data to read", instream.read() >= 0);
+ // overwrite
+ writeDataset(fs, testpath, shortDataset, shortDataset.length, 1024, true);
+ // here the file length is less. Probe the file to see if this is true,
+ // with a spin and wait
+ eventually(30 *1000, new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ assertEquals(shortLen, fs.getFileStatus(testpath).getLen());
+ return null;
+ }
+ });
+ // here length is shorter. Assuming it has propagated to all replicas,
+ // the position of the input stream is now beyond the EOF.
+ // An attempt to seek backwards to a position greater than the
+ // short length will raise an exception from AWS S3, which must be
+ // translated into an EOF
+
+ instream.seek(shortLen + 1024);
+ int c = instream.read();
+ assertIsEOF("read()", c);
+
+ byte[] buf = new byte[256];
+
+ assertIsEOF("read(buffer)", instream.read(buf));
+ assertIsEOF("read(offset)",
+ instream.read(instream.getPos(), buf, 0, buf.length));
+
+ // now do a block read fully, again, backwards from the current pos
+ try {
+ instream.readFully(shortLen + 512, buf);
+ fail("Expected readFully to fail");
+ } catch (EOFException expected) {
+ LOG.debug("Expected EOF: ", expected);
+ }
+
+ assertIsEOF("read(offset)",
+ instream.read(shortLen + 510, buf, 0, buf.length));
+
+ // seek somewhere useful
+ instream.seek(shortLen - 256);
+
+ // delete the file. Reads must fail
+ fs.delete(testpath, false);
+
+ try {
+ int r = instream.read();
+ fail("Expected an exception, got " + r);
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+
+ try {
+ instream.readFully(2048, buf);
+ fail("Expected readFully to fail");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+
+ }
+ }
+
+ /**
+ * Assert that a read operation returned an EOF value.
+ * @param operation specific operation
+ * @param readResult result
+ */
+ private void assertIsEOF(String operation, int readResult) {
+ assertEquals("Expected EOF from "+ operation
+ + "; got char " + (char) readResult, -1, readResult);
+ }
+
+ @Test
+ public void test404isNotFound() throws Throwable {
+ verifyTranslated(FileNotFoundException.class, createS3Exception(404));
+ }
+
+ protected Exception verifyTranslated(Class clazz,
+ AmazonClientException exception) throws Exception {
+ return verifyExceptionClass(clazz,
+ translateException("test", "/", exception));
+ }
+
+ @Test
+ public void test401isNotPermittedFound() throws Throwable {
+ verifyTranslated(AccessDeniedException.class,
+ createS3Exception(401));
+ }
+
+ protected AmazonS3Exception createS3Exception(int code) {
+ AmazonS3Exception source = new AmazonS3Exception("");
+ source.setStatusCode(code);
+ return source;
+ }
+
+ @Test
+ public void testGenericS3Exception() throws Throwable {
+ // S3 exception of no known type
+ AWSS3IOException ex = (AWSS3IOException)verifyTranslated(
+ AWSS3IOException.class,
+ createS3Exception(451));
+ assertEquals(451, ex.getStatusCode());
+ }
+
+ @Test
+ public void testGenericServiceS3Exception() throws Throwable {
+ // service exception of no known type
+ AmazonServiceException ase = new AmazonServiceException("unwind");
+ ase.setStatusCode(500);
+ AWSServiceIOException ex = (AWSServiceIOException)verifyTranslated(
+ AWSServiceIOException.class,
+ ase);
+ assertEquals(500, ex.getStatusCode());
+ }
+
+ @Test
+ public void testGenericClientException() throws Throwable {
+ // Generic Amazon exception
+ verifyTranslated(AWSClientIOException.class,
+ new AmazonClientException(""));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFastOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFastOutputStream.java
new file mode 100644
index 0000000..b5fa1c3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFastOutputStream.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+/**
+ * Tests regular and multi-part upload functionality for S3AFastOutputStream.
+ * File sizes are kept small to reduce test duration on slow connections
+ */
+public class ITestS3AFastOutputStream {
+ private FileSystem fs;
+
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024);
+ conf.setInt(Constants.MULTIPART_SIZE, 5 * 1024 * 1024);
+ conf.setBoolean(Constants.FAST_UPLOAD, true);
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(getTestPath(), true);
+ }
+ }
+
+ protected Path getTestPath() {
+ return new Path("/tests3a");
+ }
+
+ @Test
+ public void testRegularUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
+ }
+
+ @Test
+ public void testMultiPartUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 *
+ 1024);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
new file mode 100644
index 0000000..2a6ba0c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.net.URI;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Statistic.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.MetricDiff;
+import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
+
+/**
+ * Use metrics to assert about the cost of file status queries.
+ * {@link S3AFileSystem#getFileStatus(Path)}.
+ */
+public class ITestS3AFileOperationCost extends AbstractFSContractTestBase {
+
+ private MetricDiff metadataRequests;
+ private MetricDiff listRequests;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AFileOperationCost.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Override
+ public S3AFileSystem getFileSystem() {
+ return (S3AFileSystem) super.getFileSystem();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ S3AFileSystem fs = getFileSystem();
+ metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
+ listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnFile() throws Throwable {
+ describe("performing getFileStatus on a file");
+ Path simpleFile = path("simple.txt");
+ S3AFileSystem fs = getFileSystem();
+ touch(fs, simpleFile);
+ resetMetricDiffs();
+ S3AFileStatus status = fs.getFileStatus(simpleFile);
+ assertTrue("not a file: " + status, status.isFile());
+ metadataRequests.assertDiffEquals(1);
+ listRequests.assertDiffEquals(0);
+ }
+
+ private void resetMetricDiffs() {
+ reset(metadataRequests, listRequests);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnEmptyDir() throws Throwable {
+ describe("performing getFileStatus on an empty directory");
+ S3AFileSystem fs = getFileSystem();
+ Path dir = path("empty");
+ fs.mkdirs(dir);
+ resetMetricDiffs();
+ S3AFileStatus status = fs.getFileStatus(dir);
+ assertTrue("not empty: " + status, status.isEmptyDirectory());
+ metadataRequests.assertDiffEquals(2);
+ listRequests.assertDiffEquals(0);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnMissingFile() throws Throwable {
+ describe("performing getFileStatus on a missing file");
+ S3AFileSystem fs = getFileSystem();
+ Path path = path("missing");
+ resetMetricDiffs();
+ try {
+ S3AFileStatus status = fs.getFileStatus(path);
+ fail("Got a status back from a missing file path " + status);
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+ metadataRequests.assertDiffEquals(2);
+ listRequests.assertDiffEquals(1);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnMissingSubPath() throws Throwable {
+ describe("performing getFileStatus on a missing file");
+ S3AFileSystem fs = getFileSystem();
+ Path path = path("missingdir/missingpath");
+ resetMetricDiffs();
+ try {
+ S3AFileStatus status = fs.getFileStatus(path);
+ fail("Got a status back from a missing file path " + status);
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+ metadataRequests.assertDiffEquals(2);
+ listRequests.assertDiffEquals(1);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnNonEmptyDir() throws Throwable {
+ describe("performing getFileStatus on a non-empty directory");
+ S3AFileSystem fs = getFileSystem();
+ Path dir = path("empty");
+ fs.mkdirs(dir);
+ Path simpleFile = new Path(dir, "simple.txt");
+ touch(fs, simpleFile);
+ resetMetricDiffs();
+ S3AFileStatus status = fs.getFileStatus(dir);
+ if (status.isEmptyDirectory()) {
+ // erroneous state
+ String fsState = fs.toString();
+ fail("FileStatus says directory isempty: " + status
+ + "\n" + ContractTestUtils.ls(fs, dir)
+ + "\n" + fsState);
+ }
+ metadataRequests.assertDiffEquals(2);
+ listRequests.assertDiffEquals(1);
+ }
+
+ @Test
+ public void testCostOfCopyFromLocalFile() throws Throwable {
+ describe("testCostOfCopyFromLocalFile");
+ File localTestDir = getTestDir("tmp");
+ localTestDir.mkdirs();
+ File tmpFile = File.createTempFile("tests3acost", ".txt",
+ localTestDir);
+ tmpFile.delete();
+ try {
+ URI localFileURI = tmpFile.toURI();
+ FileSystem localFS = FileSystem.get(localFileURI,
+ getFileSystem().getConf());
+ Path localPath = new Path(localFileURI);
+ int len = 10 * 1024;
+ byte[] data = dataset(len, 'A', 'Z');
+ writeDataset(localFS, localPath, data, len, 1024, true);
+ S3AFileSystem s3a = getFileSystem();
+ MetricDiff copyLocalOps = new MetricDiff(s3a,
+ INVOCATION_COPY_FROM_LOCAL_FILE);
+ MetricDiff putRequests = new MetricDiff(s3a,
+ OBJECT_PUT_REQUESTS);
+ MetricDiff putBytes = new MetricDiff(s3a,
+ OBJECT_PUT_BYTES);
+
+ Path remotePath = path("copied");
+ s3a.copyFromLocalFile(false, true, localPath, remotePath);
+ verifyFileContents(s3a, remotePath, data);
+ copyLocalOps.assertDiffEquals(1);
+ putRequests.assertDiffEquals(1);
+ putBytes.assertDiffEquals(len);
+ // print final stats
+ LOG.info("Filesystem {}", s3a);
+ } finally {
+ tmpFile.delete();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
new file mode 100644
index 0000000..858ac22
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Tests a live S3 system. If your keys and bucket aren't specified, all tests
+ * are marked as passed.
+ *
+ * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
+ * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
+ * properly making it impossible to skip the tests if we don't have a valid
+ * bucket.
+ **/
+public class ITestS3AFileSystemContract extends FileSystemContractBaseTest {
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AFileSystemContract.class);
+
+ @Override
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ super.setUp();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(path("test"), true);
+ }
+ super.tearDown();
+ }
+
+ @Override
+ public void testMkdirsWithUmask() throws Exception {
+ // not supported
+ }
+
+ @Override
+ public void testRenameFileAsExistingFile() throws Exception {
+ if (!renameSupported()) {
+ return;
+ }
+
+ Path src = path("/test/hadoop/file");
+ createFile(src);
+ Path dst = path("/test/new/newfile");
+ createFile(dst);
+ // s3 doesn't support rename option
+ // rename-overwrites-dest is always allowed.
+ rename(src, dst, true, false, true);
+ }
+
+ @Override
+ public void testRenameDirectoryAsExistingDirectory() throws Exception {
+ if (!renameSupported()) {
+ return;
+ }
+
+ Path src = path("/test/hadoop/dir");
+ fs.mkdirs(src);
+ createFile(path("/test/hadoop/dir/file1"));
+ createFile(path("/test/hadoop/dir/subdir/file2"));
+
+ Path dst = path("/test/new/newdir");
+ fs.mkdirs(dst);
+ rename(src, dst, true, false, true);
+ assertFalse("Nested file1 exists",
+ fs.exists(path("/test/hadoop/dir/file1")));
+ assertFalse("Nested file2 exists",
+ fs.exists(path("/test/hadoop/dir/subdir/file2")));
+ assertTrue("Renamed nested file1 exists",
+ fs.exists(path("/test/new/newdir/file1")));
+ assertTrue("Renamed nested exists",
+ fs.exists(path("/test/new/newdir/subdir/file2")));
+ }
+
+// @Override
+ public void testMoveDirUnderParent() throws Throwable {
+ // not support because
+ // Fails if dst is a directory that is not empty.
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
new file mode 100644
index 0000000..360a151
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.net.URI;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
+import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest;
+import com.amazonaws.services.securitytoken.model.GetSessionTokenResult;
+import com.amazonaws.services.securitytoken.model.Credentials;
+
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3native.S3xLoginHelper;
+import org.apache.hadoop.conf.Configuration;
+
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * Tests use of temporary credentials (for example, AWS STS & S3).
+ * This test extends a class that "does things to the root directory", and
+ * should only be used against transient filesystems where you don't care about
+ * the data.
+ */
+public class ITestS3ATemporaryCredentials extends AbstractFSContractTestBase {
+ public static final String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled";
+ public static final String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3ATemporaryCredentials.class);
+
+ private static final String PROVIDER_CLASS
+ = TemporaryAWSCredentialsProvider.NAME;
+
+ private static final long TEST_FILE_SIZE = 1024;
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ /**
+ * Test use of STS for requesting temporary credentials.
+ *
+ * The property test.sts.endpoint can be set to point this at different
+ * STS endpoints. This test will use the AWS credentials (if provided) for
+ * S3A tests to request temporary credentials, then attempt to use those
+ * credentials instead.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testSTS() throws IOException {
+ Configuration conf = getContract().getConf();
+ if (!conf.getBoolean(TEST_STS_ENABLED, true)) {
+ skip("STS functional tests disabled");
+ }
+
+ S3xLoginHelper.Login login = S3AUtils.getAWSAccessKeys(
+ URI.create("s3a://foobar"), conf);
+ if (!login.hasLogin()) {
+ skip("testSTS disabled because AWS credentials not configured");
+ }
+ AWSCredentialsProvider parentCredentials = new BasicAWSCredentialsProvider(
+ login.getUser(), login.getPassword());
+
+ String stsEndpoint = conf.getTrimmed(TEST_STS_ENDPOINT, "");
+ AWSSecurityTokenServiceClient stsClient;
+ stsClient = new AWSSecurityTokenServiceClient(parentCredentials);
+ if (!stsEndpoint.isEmpty()) {
+ LOG.debug("STS Endpoint ={}", stsEndpoint);
+ stsClient.setEndpoint(stsEndpoint);
+ }
+ GetSessionTokenRequest sessionTokenRequest = new GetSessionTokenRequest();
+ sessionTokenRequest.setDurationSeconds(900);
+ GetSessionTokenResult sessionTokenResult;
+ sessionTokenResult = stsClient.getSessionToken(sessionTokenRequest);
+ Credentials sessionCreds = sessionTokenResult.getCredentials();
+
+ String childAccessKey = sessionCreds.getAccessKeyId();
+ conf.set(ACCESS_KEY, childAccessKey);
+ String childSecretKey = sessionCreds.getSecretAccessKey();
+ conf.set(SECRET_KEY, childSecretKey);
+ String sessionToken = sessionCreds.getSessionToken();
+ conf.set(SESSION_TOKEN, sessionToken);
+
+ conf.set(AWS_CREDENTIALS_PROVIDER, PROVIDER_CLASS);
+
+ try(S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
+ createAndVerifyFile(fs, path("testSTS"), TEST_FILE_SIZE);
+ }
+
+ // now create an invalid set of credentials by changing the session
+ // token
+ conf.set(SESSION_TOKEN, "invalid-" + sessionToken);
+ try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
+ createAndVerifyFile(fs, path("testSTSInvalidToken"), TEST_FILE_SIZE);
+ fail("Expected an access exception, but file access to "
+ + fs.getUri() + " was allowed: " + fs);
+ } catch (AWSS3IOException ex) {
+ LOG.info("Expected Exception: {}", ex.toString());
+ LOG.debug("Expected Exception: {}", ex, ex);
+ }
+ }
+
+ @Test
+ public void testTemporaryCredentialValidation() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(ACCESS_KEY, "accesskey");
+ conf.set(SECRET_KEY, "secretkey");
+ conf.set(SESSION_TOKEN, "");
+ TemporaryAWSCredentialsProvider provider
+ = new TemporaryAWSCredentialsProvider(getFileSystem().getUri(), conf);
+ try {
+ AWSCredentials credentials = provider.getCredentials();
+ fail("Expected a CredentialInitializationException,"
+ + " got " + credentials);
+ } catch (CredentialInitializationException expected) {
+ // expected
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBlockingThreadPoolExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBlockingThreadPoolExecutorService.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBlockingThreadPoolExecutorService.java
deleted file mode 100644
index 25a8958..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBlockingThreadPoolExecutorService.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import org.apache.hadoop.util.StopWatch;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-/**
- * Basic unit test for S3A's blocking executor service.
- */
-public class TestBlockingThreadPoolExecutorService {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- BlockingThreadPoolExecutorService.class);
-
- private static final int NUM_ACTIVE_TASKS = 4;
- private static final int NUM_WAITING_TASKS = 2;
- private static final int TASK_SLEEP_MSEC = 100;
- private static final int SHUTDOWN_WAIT_MSEC = 200;
- private static final int SHUTDOWN_WAIT_TRIES = 5;
- private static final int BLOCKING_THRESHOLD_MSEC = 50;
-
- private static final Integer SOME_VALUE = 1337;
-
- private static BlockingThreadPoolExecutorService tpe = null;
-
- @AfterClass
- public static void afterClass() throws Exception {
- ensureDestroyed();
- }
-
- /**
- * Basic test of running one trivial task.
- */
- @Test
- public void testSubmitCallable() throws Exception {
- ensureCreated();
- ListenableFuture<Integer> f = tpe.submit(callableSleeper);
- Integer v = f.get();
- assertEquals(SOME_VALUE, v);
- }
-
- /**
- * More involved test, including detecting blocking when at capacity.
- */
- @Test
- public void testSubmitRunnable() throws Exception {
- ensureCreated();
- int totalTasks = NUM_ACTIVE_TASKS + NUM_WAITING_TASKS;
- StopWatch stopWatch = new StopWatch().start();
- for (int i = 0; i < totalTasks; i++) {
- tpe.submit(sleeper);
- assertDidntBlock(stopWatch);
- }
- tpe.submit(sleeper);
- assertDidBlock(stopWatch);
- }
-
- @Test
- public void testShutdown() throws Exception {
- // Cover create / destroy, regardless of when this test case runs
- ensureCreated();
- ensureDestroyed();
-
- // Cover create, execute, destroy, regardless of when test case runs
- ensureCreated();
- testSubmitRunnable();
- ensureDestroyed();
- }
-
- // Helper functions, etc.
-
- private void assertDidntBlock(StopWatch sw) {
- try {
- assertFalse("Non-blocking call took too long.",
- sw.now(TimeUnit.MILLISECONDS) > BLOCKING_THRESHOLD_MSEC);
- } finally {
- sw.reset().start();
- }
- }
-
- private void assertDidBlock(StopWatch sw) {
- try {
- if (sw.now(TimeUnit.MILLISECONDS) < BLOCKING_THRESHOLD_MSEC) {
- throw new RuntimeException("Blocking call returned too fast.");
- }
- } finally {
- sw.reset().start();
- }
- }
-
- private Runnable sleeper = new Runnable() {
- @Override
- public void run() {
- String name = Thread.currentThread().getName();
- try {
- Thread.sleep(TASK_SLEEP_MSEC);
- } catch (InterruptedException e) {
- LOG.info("Thread {} interrupted.", name);
- Thread.currentThread().interrupt();
- }
- }
- };
-
- private Callable<Integer> callableSleeper = new Callable<Integer>() {
- @Override
- public Integer call() throws Exception {
- sleeper.run();
- return SOME_VALUE;
- }
- };
-
- /**
- * Helper function to create thread pool under test.
- */
- private static void ensureCreated() throws Exception {
- if (tpe == null) {
- LOG.debug("Creating thread pool");
- tpe = new BlockingThreadPoolExecutorService(NUM_ACTIVE_TASKS,
- NUM_WAITING_TASKS, 1, TimeUnit.SECONDS, "btpetest");
- }
- }
-
- /**
- * Helper function to terminate thread pool under test, asserting that
- * shutdown -> terminate works as expected.
- */
- private static void ensureDestroyed() throws Exception {
- if (tpe == null) {
- return;
- }
- int shutdownTries = SHUTDOWN_WAIT_TRIES;
-
- tpe.shutdown();
- if (!tpe.isShutdown()) {
- throw new RuntimeException("Shutdown had no effect.");
- }
-
- while (!tpe.awaitTermination(SHUTDOWN_WAIT_MSEC,
- TimeUnit.MILLISECONDS)) {
- LOG.info("Waiting for thread pool shutdown.");
- if (shutdownTries-- <= 0) {
- LOG.error("Failed to terminate thread pool gracefully.");
- break;
- }
- }
- if (!tpe.isTerminated()) {
- tpe.shutdownNow();
- if (!tpe.awaitTermination(SHUTDOWN_WAIT_MSEC,
- TimeUnit.MILLISECONDS)) {
- throw new RuntimeException(
- "Failed to terminate thread pool in timely manner.");
- }
- }
- tpe = null;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDFS-8986. Add option to -du to
calculate directory space usage excluding snapshots. Contributed by Xiao
Chen.
Posted by cd...@apache.org.
HDFS-8986. Add option to -du to calculate directory space usage excluding snapshots. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0efea49
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0efea49
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0efea49
Branch: refs/heads/HDFS-9806
Commit: f0efea490e5aa9dd629d2199aae9c5b1290a17ee
Parents: dd76238
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Tue Aug 23 04:13:48 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Tue Aug 23 04:14:24 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/ContentSummary.java | 127 ++++++++--
.../java/org/apache/hadoop/fs/shell/Count.java | 20 +-
.../org/apache/hadoop/fs/shell/FsUsage.java | 31 ++-
.../src/site/markdown/FileSystemShell.md | 11 +-
.../org/apache/hadoop/fs/shell/TestCount.java | 5 +-
.../src/test/resources/testConf.xml | 12 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 8 +
.../src/main/proto/hdfs.proto | 4 +
.../ContentSummaryComputationContext.java | 6 +
.../hadoop/hdfs/server/namenode/INode.java | 9 +-
.../hdfs/server/namenode/INodeDirectory.java | 4 +
.../org/apache/hadoop/hdfs/TestDFSShell.java | 235 ++++++++++++++++++-
12 files changed, 432 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index 3dedbcc..3e75951 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -34,6 +34,11 @@ public class ContentSummary extends QuotaUsage implements Writable{
private long length;
private long fileCount;
private long directoryCount;
+ // These fields are to track the snapshot-related portion of the values.
+ private long snapshotLength;
+ private long snapshotFileCount;
+ private long snapshotDirectoryCount;
+ private long snapshotSpaceConsumed;
/** We don't use generics. Instead override spaceConsumed and other methods
in order to keep backward compatibility. */
@@ -56,6 +61,26 @@ public class ContentSummary extends QuotaUsage implements Writable{
return this;
}
+ public Builder snapshotLength(long snapshotLength) {
+ this.snapshotLength = snapshotLength;
+ return this;
+ }
+
+ public Builder snapshotFileCount(long snapshotFileCount) {
+ this.snapshotFileCount = snapshotFileCount;
+ return this;
+ }
+
+ public Builder snapshotDirectoryCount(long snapshotDirectoryCount) {
+ this.snapshotDirectoryCount = snapshotDirectoryCount;
+ return this;
+ }
+
+ public Builder snapshotSpaceConsumed(long snapshotSpaceConsumed) {
+ this.snapshotSpaceConsumed = snapshotSpaceConsumed;
+ return this;
+ }
+
@Override
public Builder quota(long quota){
super.quota(quota);
@@ -107,6 +132,10 @@ public class ContentSummary extends QuotaUsage implements Writable{
private long length;
private long fileCount;
private long directoryCount;
+ private long snapshotLength;
+ private long snapshotFileCount;
+ private long snapshotDirectoryCount;
+ private long snapshotSpaceConsumed;
}
/** Constructor deprecated by ContentSummary.Builder*/
@@ -142,17 +171,37 @@ public class ContentSummary extends QuotaUsage implements Writable{
this.length = builder.length;
this.fileCount = builder.fileCount;
this.directoryCount = builder.directoryCount;
+ this.snapshotLength = builder.snapshotLength;
+ this.snapshotFileCount = builder.snapshotFileCount;
+ this.snapshotDirectoryCount = builder.snapshotDirectoryCount;
+ this.snapshotSpaceConsumed = builder.snapshotSpaceConsumed;
}
/** @return the length */
public long getLength() {return length;}
+ public long getSnapshotLength() {
+ return snapshotLength;
+ }
+
/** @return the directory count */
public long getDirectoryCount() {return directoryCount;}
+ public long getSnapshotDirectoryCount() {
+ return snapshotDirectoryCount;
+ }
+
/** @return the file count */
public long getFileCount() {return fileCount;}
+ public long getSnapshotFileCount() {
+ return snapshotFileCount;
+ }
+
+ public long getSnapshotSpaceConsumed() {
+ return snapshotSpaceConsumed;
+ }
+
@Override
@InterfaceAudience.Private
public void write(DataOutput out) throws IOException {
@@ -180,9 +229,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
if (this == to) {
return true;
} else if (to instanceof ContentSummary) {
- return getLength() == ((ContentSummary) to).getLength() &&
- getFileCount() == ((ContentSummary) to).getFileCount() &&
- getDirectoryCount() == ((ContentSummary) to).getDirectoryCount() &&
+ ContentSummary right = (ContentSummary) to;
+ return getLength() == right.getLength() &&
+ getFileCount() == right.getFileCount() &&
+ getDirectoryCount() == right.getDirectoryCount() &&
+ getSnapshotLength() == right.getSnapshotLength() &&
+ getSnapshotFileCount() == right.getSnapshotFileCount() &&
+ getSnapshotDirectoryCount() == right.getSnapshotDirectoryCount() &&
+ getSnapshotSpaceConsumed() == right.getSnapshotSpaceConsumed() &&
super.equals(to);
} else {
return super.equals(to);
@@ -191,7 +245,9 @@ public class ContentSummary extends QuotaUsage implements Writable{
@Override
public int hashCode() {
- long result = getLength() ^ getFileCount() ^ getDirectoryCount();
+ long result = getLength() ^ getFileCount() ^ getDirectoryCount()
+ ^ getSnapshotLength() ^ getSnapshotFileCount()
+ ^ getSnapshotDirectoryCount() ^ getSnapshotSpaceConsumed();
return ((int) result) ^ super.hashCode();
}
@@ -255,15 +311,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
* @param qOption a flag indicating if quota needs to be printed or not
* @return the string representation of the object
*/
+ @Override
public String toString(boolean qOption) {
return toString(qOption, false);
}
/** Return the string representation of the object in the output format.
- * if qOption is false, output directory count, file count, and content size;
- * if qOption is true, output quota and remaining quota as well.
- * if hOption is false file sizes are returned in bytes
- * if hOption is true file sizes are returned in human readable
+ * For description of the options,
+ * @see #toString(boolean, boolean, boolean, boolean, List)
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output if to be used
@@ -273,10 +328,24 @@ public class ContentSummary extends QuotaUsage implements Writable{
return toString(qOption, hOption, false, null);
}
+ /** Return the string representation of the object in the output format.
+ * For description of the options,
+ * @see #toString(boolean, boolean, boolean, boolean, List)
+ *
+ * @param qOption a flag indicating if quota needs to be printed or not
+ * @param hOption a flag indicating if human readable output is to be used
+ * @param xOption a flag indicating if calculation from snapshots is to be
+ * included in the output
+ * @return the string representation of the object
+ */
+ public String toString(boolean qOption, boolean hOption, boolean xOption) {
+ return toString(qOption, hOption, false, xOption, null);
+ }
+
/**
* Return the string representation of the object in the output format.
- * if tOption is true, display the quota by storage types,
- * Otherwise, same logic with #toString(boolean,boolean)
+ * For description of the options,
+ * @see #toString(boolean, boolean, boolean, boolean, List)
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output if to be used
@@ -286,6 +355,29 @@ public class ContentSummary extends QuotaUsage implements Writable{
*/
public String toString(boolean qOption, boolean hOption,
boolean tOption, List<StorageType> types) {
+ return toString(qOption, hOption, tOption, false, types);
+ }
+
+ /** Return the string representation of the object in the output format.
+ * if qOption is false, output directory count, file count, and content size;
+ * if qOption is true, output quota and remaining quota as well.
+ * if hOption is false, file sizes are returned in bytes
+ * if hOption is true, file sizes are returned in human readable
+ * if tOption is true, display the quota by storage types
+ * if tOption is false, same logic with #toString(boolean,boolean)
+ * if xOption is false, output includes the calculation from snapshots
+ * if xOption is true, output excludes the calculation from snapshots
+ *
+ * @param qOption a flag indicating if quota needs to be printed or not
+ * @param hOption a flag indicating if human readable output is to be used
+ * @param tOption a flag indicating if display quota by storage types
+ * @param xOption a flag indicating if calculation from snapshots is to be
+ * included in the output
+ * @param types Storage types to display
+ * @return the string representation of the object
+ */
+ public String toString(boolean qOption, boolean hOption, boolean tOption,
+ boolean xOption, List<StorageType> types) {
String prefix = "";
if (tOption) {
@@ -296,10 +388,17 @@ public class ContentSummary extends QuotaUsage implements Writable{
prefix = getQuotaUsage(hOption);
}
- return prefix + String.format(SUMMARY_FORMAT,
- formatSize(directoryCount, hOption),
- formatSize(fileCount, hOption),
- formatSize(length, hOption));
+ if (xOption) {
+ return prefix + String.format(SUMMARY_FORMAT,
+ formatSize(directoryCount - snapshotDirectoryCount, hOption),
+ formatSize(fileCount - snapshotFileCount, hOption),
+ formatSize(length - snapshotLength, hOption));
+ } else {
+ return prefix + String.format(SUMMARY_FORMAT,
+ formatSize(directoryCount, hOption),
+ formatSize(fileCount, hOption),
+ formatSize(length, hOption));
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index 2b5a4cc..16f52e3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -51,6 +51,8 @@ public class Count extends FsCommand {
private static final String OPTION_HUMAN = "h";
private static final String OPTION_HEADER = "v";
private static final String OPTION_TYPE = "t";
+ // exclude snapshots from calculation. Only work on default columns.
+ private static final String OPTION_EXCLUDE_SNAPSHOT = "x";
//return the quota, namespace count and disk space usage.
private static final String OPTION_QUOTA_AND_USAGE = "u";
@@ -58,7 +60,8 @@ public class Count extends FsCommand {
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
- OPTION_QUOTA_AND_USAGE + "] <path> ...";
+ OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
+ + "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" +
@@ -72,6 +75,8 @@ public class Count extends FsCommand {
"The -" + OPTION_HUMAN +
" option shows file sizes in human readable format.\n" +
"The -" + OPTION_HEADER + " option displays a header line.\n" +
+ "The -" + OPTION_EXCLUDE_SNAPSHOT + " option excludes snapshots " +
+ "from being calculated. \n" +
"The -" + OPTION_TYPE + " option displays quota by storage types.\n" +
"It must be used with -" + OPTION_QUOTA + " option.\n" +
"If a comma-separated list of storage types is given after the -" +
@@ -87,6 +92,7 @@ public class Count extends FsCommand {
private boolean showQuotabyType;
private List<StorageType> storageTypes = null;
private boolean showQuotasAndUsageOnly;
+ private boolean excludeSnapshots;
/** Constructor */
public Count() {}
@@ -106,7 +112,8 @@ public class Count extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
- OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE);
+ OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
+ OPTION_EXCLUDE_SNAPSHOT);
cf.addOptionWithValue(OPTION_TYPE);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
@@ -115,6 +122,7 @@ public class Count extends FsCommand {
showQuotas = cf.getOpt(OPTION_QUOTA);
humanReadable = cf.getOpt(OPTION_HUMAN);
showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
+ excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
if (showQuotas || showQuotasAndUsageOnly) {
String types = cf.getOptValue(OPTION_TYPE);
@@ -125,6 +133,11 @@ public class Count extends FsCommand {
} else {
showQuotabyType = false;
}
+ if (excludeSnapshots) {
+ out.println(OPTION_QUOTA + " or " + OPTION_QUOTA_AND_USAGE + " option "
+ + "is given, the -x option is ignored.");
+ excludeSnapshots = false;
+ }
}
if (cf.getOpt(OPTION_HEADER)) {
@@ -163,7 +176,8 @@ public class Count extends FsCommand {
storageTypes) + src);
} else {
ContentSummary summary = src.fs.getContentSummary(src.path);
- out.println(summary.toString(showQuotas, isHumanReadable()) + src);
+ out.println(summary.
+ toString(showQuotas, isHumanReadable(), excludeSnapshots) + src);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
index 765b181..197920f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
@@ -107,27 +107,30 @@ class FsUsage extends FsCommand {
/** show disk usage */
public static class Du extends FsUsage {
public static final String NAME = "du";
- public static final String USAGE = "[-s] [-h] <path> ...";
+ public static final String USAGE = "[-s] [-h] [-x] <path> ...";
public static final String DESCRIPTION =
- "Show the amount of space, in bytes, used by the files that " +
- "match the specified file pattern. The following flags are optional:\n" +
- "-s: Rather than showing the size of each individual file that" +
- " matches the pattern, shows the total (summary) size.\n" +
- "-h: Formats the sizes of files in a human-readable fashion" +
- " rather than a number of bytes.\n\n" +
- "Note that, even without the -s option, this only shows size summaries " +
- "one level deep into a directory.\n\n" +
- "The output is in the form \n" +
- "\tsize\tdisk space consumed\tname(full path)\n";
+ "Show the amount of space, in bytes, used by the files that match " +
+ "the specified file pattern. The following flags are optional:\n" +
+ "-s: Rather than showing the size of each individual file that" +
+ " matches the pattern, shows the total (summary) size.\n" +
+ "-h: Formats the sizes of files in a human-readable fashion" +
+ " rather than a number of bytes.\n" +
+ "-x: Excludes snapshots from being counted.\n\n" +
+ "Note that, even without the -s option, this only shows size " +
+ "summaries one level deep into a directory.\n\n" +
+ "The output is in the form \n" +
+ "\tsize\tdisk space consumed\tname(full path)\n";
protected boolean summary = false;
+ private boolean excludeSnapshots = false;
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
- CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s");
+ CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "x");
cf.parse(args);
humanReadable = cf.getOpt("h");
summary = cf.getOpt("s");
+ excludeSnapshots = cf.getOpt("x");
if (args.isEmpty()) args.add(Path.CUR_DIR);
}
@@ -156,6 +159,10 @@ class FsUsage extends FsCommand {
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
long length = contentSummary.getLength();
long spaceConsumed = contentSummary.getSpaceConsumed();
+ if (excludeSnapshots) {
+ length -= contentSummary.getSnapshotLength();
+ spaceConsumed -= contentSummary.getSnapshotSpaceConsumed();
+ }
usagesTable.addRow(formatSize(length), formatSize(spaceConsumed), item);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 066cfe3..a8a1920 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -132,10 +132,12 @@ Similar to get command, except that the destination is restricted to a local fil
count
-----
-Usage: `hadoop fs -count [-q] [-h] [-v] [-t [<storage type>]] [-u] <paths> `
+Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] <paths> `
Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+The -u and -q options control what columns the output contains. -q means show quotas, -u limits the output to show quotas and usage only.
+
The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA
@@ -146,6 +148,8 @@ The -h option shows sizes in human readable format.
The -v option displays a header line.
+The -x option excludes snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path. The -x option is ignored if -u or -q option is given.
+
Example:
* `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@@ -211,14 +215,15 @@ Example:
du
----
-Usage: `hadoop fs -du [-s] [-h] URI [URI ...]`
+Usage: `hadoop fs -du [-s] [-h] [-x] URI [URI ...]`
Displays sizes of files and directories contained in the given directory or the length of a file in case its just a file.
Options:
-* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files.
+* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. Without the -s option, calculation is done by going 1-level deep from the given path.
* The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864)
+* The -x option will exclude snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path.
The du returns three columns with the following format:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
index 04174e0..c7b30e4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
@@ -447,7 +447,7 @@ public class TestCount {
Count count = new Count();
String actual = count.getUsage();
String expected =
- "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] <path> ...";
+ "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
@@ -465,6 +465,7 @@ public class TestCount {
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "The -h option shows file sizes in human readable format.\n"
+ "The -v option displays a header line.\n"
+ + "The -x option excludes snapshots from being calculated. \n"
+ "The -t option displays quota by storage types.\n"
+ "It must be used with -q option.\n"
+ "If a comma-separated list of storage types is given after the -t option, \n"
@@ -521,7 +522,7 @@ public class TestCount {
}
@Override
- public String toString(boolean qOption, boolean hOption) {
+ public String toString(boolean qOption, boolean hOption, boolean xOption) {
if (qOption) {
if (hOption) {
return (HUMAN + WITH_QUOTAS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 82bc789..a31c828 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -200,7 +200,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-du \[-s\] \[-h\] <path> \.\.\. :\s*</expected-output>
+ <expected-output>^-du \[-s\] \[-h\] \[-x\] <path> \.\.\. :\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -228,6 +228,10 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
+ <expected-output>^\s*-x\s*Excludes snapshots from being counted.\s*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
<expected-output>^\s*Note that, even without the -s option, this only shows size summaries one level\s*</expected-output>
</comparator>
<comparator>
@@ -274,7 +278,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] <path> \.\.\. :( )*</expected-output>
+ <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] \[-x\] <path> \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -308,6 +312,10 @@
<type>RegexpComparator</type>
<expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
</comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*The -x option excludes snapshots from being calculated.( )*</expected-output>
+ </comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 3c5f583..7df68e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -1466,6 +1466,10 @@ public class PBHelperClient {
builder.length(cs.getLength()).
fileCount(cs.getFileCount()).
directoryCount(cs.getDirectoryCount()).
+ snapshotLength(cs.getSnapshotLength()).
+ snapshotFileCount(cs.getSnapshotFileCount()).
+ snapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
+ snapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
quota(cs.getQuota()).
spaceConsumed(cs.getSpaceConsumed()).
spaceQuota(cs.getSpaceQuota());
@@ -2069,6 +2073,10 @@ public class PBHelperClient {
builder.setLength(cs.getLength()).
setFileCount(cs.getFileCount()).
setDirectoryCount(cs.getDirectoryCount()).
+ setSnapshotLength(cs.getSnapshotLength()).
+ setSnapshotFileCount(cs.getSnapshotFileCount()).
+ setSnapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
+ setSnapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
setQuota(cs.getQuota()).
setSpaceConsumed(cs.getSpaceConsumed()).
setSpaceQuota(cs.getSpaceQuota());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index f939ec3..7de8550 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -138,6 +138,10 @@ message ContentSummaryProto {
required uint64 spaceConsumed = 5;
required uint64 spaceQuota = 6;
optional StorageTypeQuotaInfosProto typeQuotaInfos = 7;
+ optional uint64 snapshotLength = 8;
+ optional uint64 snapshotFileCount = 9;
+ optional uint64 snapshotDirectoryCount = 10;
+ optional uint64 snapshotSpaceConsumed = 11;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
index 5739835..6df9e75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
@@ -29,6 +29,7 @@ public class ContentSummaryComputationContext {
private FSNamesystem fsn = null;
private BlockStoragePolicySuite bsps = null;
private ContentCounts counts = null;
+ private ContentCounts snapshotCounts = null;
private long nextCountLimit = 0;
private long limitPerRun = 0;
private long yieldCount = 0;
@@ -51,6 +52,7 @@ public class ContentSummaryComputationContext {
this.limitPerRun = limitPerRun;
this.nextCountLimit = limitPerRun;
this.counts = new ContentCounts.Builder().build();
+ this.snapshotCounts = new ContentCounts.Builder().build();
this.sleepMilliSec = sleepMicroSec/1000;
this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000);
}
@@ -125,6 +127,10 @@ public class ContentSummaryComputationContext {
return counts;
}
+ public ContentCounts getSnapshotCounts() {
+ return snapshotCounts;
+ }
+
public BlockStoragePolicySuite getBlockStoragePolicySuite() {
Preconditions.checkState((bsps != null || fsn != null),
"BlockStoragePolicySuite must be either initialized or available via" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 0348c1f..c6258a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -428,8 +428,9 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
*/
public final ContentSummary computeAndConvertContentSummary(int snapshotId,
ContentSummaryComputationContext summary) {
- ContentCounts counts = computeContentSummary(snapshotId, summary)
- .getCounts();
+ computeContentSummary(snapshotId, summary);
+ final ContentCounts counts = summary.getCounts();
+ final ContentCounts snapshotCounts = summary.getSnapshotCounts();
final QuotaCounts q = getQuotaCounts();
return new ContentSummary.Builder().
length(counts.getLength()).
@@ -440,6 +441,10 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
spaceQuota(q.getStorageSpace()).
typeConsumed(counts.getTypeSpaces()).
typeQuota(q.getTypeSpaces().asArray()).
+ snapshotLength(snapshotCounts.getLength()).
+ snapshotFileCount(snapshotCounts.getFileCount()).
+ snapshotDirectoryCount(snapshotCounts.getDirectoryCount()).
+ snapshotSpaceConsumed(snapshotCounts.getStoragespace()).
build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
index d00c136..9a8f9b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -634,6 +634,10 @@ public class INodeDirectory extends INodeWithAdditionalFields
// computation should include all the deleted files/directories
sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
summary.getCounts());
+ // Also compute ContentSummary for snapshotCounts (So we can extract it
+ // later from the ContentSummary of all).
+ sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
+ summary.getSnapshotCounts());
}
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index dcf683e..73ebf28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -113,6 +113,11 @@ public class TestDFSShell {
return p;
}
+ static void rmr(FileSystem fs, Path p) throws IOException {
+ assertTrue(fs.delete(p, true));
+ assertFalse(fs.exists(p));
+ }
+
static File createLocalFile(File f) throws IOException {
assertTrue(!f.exists());
PrintWriter out = new PrintWriter(f);
@@ -189,7 +194,7 @@ public class TestDFSShell {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
FileSystem fs = cluster.getFileSystem();
assertTrue("Not a HDFS: " + fs.getUri(),
- fs instanceof DistributedFileSystem);
+ fs instanceof DistributedFileSystem);
try {
fs.mkdirs(new Path(new Path("parent"), "child"));
try {
@@ -224,6 +229,7 @@ public class TestDFSShell {
shell.setConf(conf);
try {
+ cluster.waitActive();
Path myPath = new Path("/test/dir");
assertTrue(fs.mkdirs(myPath));
assertTrue(fs.exists(myPath));
@@ -251,7 +257,7 @@ public class TestDFSShell {
assertTrue(val == 0);
String returnString = out.toString();
out.reset();
- // Check if size matchs as expected
+ // Check if size matches as expected
assertThat(returnString, containsString(myFileLength.toString()));
assertThat(returnString, containsString(myFileDiskUsed.toString()));
assertThat(returnString, containsString(myFile2Length.toString()));
@@ -308,7 +314,232 @@ public class TestDFSShell {
System.setOut(psBackup);
cluster.shutdown();
}
+ }
+
+ @Test (timeout = 180000)
+ public void testDuSnapshots() throws IOException {
+ final int replication = 2;
+ final Configuration conf = new HdfsConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(replication).build();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final PrintStream psBackup = System.out;
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ final PrintStream psOut = new PrintStream(out);
+ final FsShell shell = new FsShell();
+ shell.setConf(conf);
+
+ try {
+ System.setOut(psOut);
+ cluster.waitActive();
+ final Path parent = new Path("/test");
+ final Path dir = new Path(parent, "dir");
+ mkdir(dfs, dir);
+ final Path file = new Path(dir, "file");
+ writeFile(dfs, file);
+ final Path file2 = new Path(dir, "file2");
+ writeFile(dfs, file2);
+ final Long fileLength = dfs.getFileStatus(file).getLen();
+ final Long fileDiskUsed = fileLength * replication;
+ final Long file2Length = dfs.getFileStatus(file2).getLen();
+ final Long file2DiskUsed = file2Length * replication;
+
+ /*
+ * Construct dir as follows:
+ * /test/dir/file <- this will later be deleted after snapshot taken.
+ * /test/dir/newfile <- this will be created after snapshot taken.
+ * /test/dir/file2
+ * Snapshot enabled on /test
+ */
+
+ // test -du on /test/dir
+ int ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", dir.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ String returnString = out.toString();
+ LOG.info("-du return is:\n" + returnString);
+ // Check if size matches as expected
+ assertTrue(returnString.contains(fileLength.toString()));
+ assertTrue(returnString.contains(fileDiskUsed.toString()));
+ assertTrue(returnString.contains(file2Length.toString()));
+ assertTrue(returnString.contains(file2DiskUsed.toString()));
+ out.reset();
+
+ // take a snapshot, then remove file and add newFile
+ final String snapshotName = "ss1";
+ final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
+ dfs.allowSnapshot(parent);
+ assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
+ rmr(dfs, file);
+ final Path newFile = new Path(dir, "newfile");
+ writeFile(dfs, newFile);
+ final Long newFileLength = dfs.getFileStatus(newFile).getLen();
+ final Long newFileDiskUsed = newFileLength * replication;
+
+ // test -du -s on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", "-s", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du -s return is:\n" + returnString);
+ Long combinedLength = fileLength + file2Length + newFileLength;
+ Long combinedDiskUsed = fileDiskUsed + file2DiskUsed + newFileDiskUsed;
+ assertTrue(returnString.contains(combinedLength.toString()));
+ assertTrue(returnString.contains(combinedDiskUsed.toString()));
+ out.reset();
+
+ // test -du on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du return is:\n" + returnString);
+ assertTrue(returnString.contains(combinedLength.toString()));
+ assertTrue(returnString.contains(combinedDiskUsed.toString()));
+ out.reset();
+
+ // test -du -s -x on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", "-s", "-x", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du -s -x return is:\n" + returnString);
+ Long exludeSnapshotLength = file2Length + newFileLength;
+ Long excludeSnapshotDiskUsed = file2DiskUsed + newFileDiskUsed;
+ assertTrue(returnString.contains(exludeSnapshotLength.toString()));
+ assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString()));
+ out.reset();
+ // test -du -x on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", "-x", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du -x return is:\n" + returnString);
+ assertTrue(returnString.contains(exludeSnapshotLength.toString()));
+ assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString()));
+ out.reset();
+ } finally {
+ System.setOut(psBackup);
+ cluster.shutdown();
+ }
+ }
+
+ @Test (timeout = 180000)
+ public void testCountSnapshots() throws IOException {
+ final int replication = 2;
+ final Configuration conf = new HdfsConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(replication).build();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final PrintStream psBackup = System.out;
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ final PrintStream psOut = new PrintStream(out);
+ System.setOut(psOut);
+ final FsShell shell = new FsShell();
+ shell.setConf(conf);
+
+ try {
+ cluster.waitActive();
+ final Path parent = new Path("/test");
+ final Path dir = new Path(parent, "dir");
+ mkdir(dfs, dir);
+ final Path file = new Path(dir, "file");
+ writeFile(dfs, file);
+ final Path file2 = new Path(dir, "file2");
+ writeFile(dfs, file2);
+ final long fileLength = dfs.getFileStatus(file).getLen();
+ final long file2Length = dfs.getFileStatus(file2).getLen();
+ final Path dir2 = new Path(parent, "dir2");
+ mkdir(dfs, dir2);
+
+ /*
+ * Construct dir as follows:
+ * /test/dir/file <- this will later be deleted after snapshot taken.
+ * /test/dir/newfile <- this will be created after snapshot taken.
+ * /test/dir/file2
+ * /test/dir2 <- this will later be deleted after snapshot taken.
+ * Snapshot enabled on /test
+ */
+
+ // take a snapshot
+ // then create /test/dir/newfile and remove /test/dir/file, /test/dir2
+ final String snapshotName = "s1";
+ final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
+ dfs.allowSnapshot(parent);
+ assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
+ rmr(dfs, file);
+ rmr(dfs, dir2);
+ final Path newFile = new Path(dir, "new file");
+ writeFile(dfs, newFile);
+ final Long newFileLength = dfs.getFileStatus(newFile).getLen();
+
+ // test -count on /test. Include header for easier debugging.
+ int val = -1;
+ try {
+ val = shell.run(new String[] {"-count", "-v", parent.toString() });
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, val);
+ String returnString = out.toString();
+ LOG.info("-count return is:\n" + returnString);
+ Scanner in = new Scanner(returnString);
+ in.nextLine();
+ assertEquals(3, in.nextLong()); //DIR_COUNT
+ assertEquals(3, in.nextLong()); //FILE_COUNT
+ assertEquals(fileLength + file2Length + newFileLength,
+ in.nextLong()); //CONTENT_SIZE
+ out.reset();
+
+ // test -count -x on /test. Include header for easier debugging.
+ val = -1;
+ try {
+ val =
+ shell.run(new String[] {"-count", "-x", "-v", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, val);
+ returnString = out.toString();
+ LOG.info("-count -x return is:\n" + returnString);
+ in = new Scanner(returnString);
+ in.nextLine();
+ assertEquals(2, in.nextLong()); //DIR_COUNT
+ assertEquals(2, in.nextLong()); //FILE_COUNT
+ assertEquals(file2Length + newFileLength, in.nextLong()); //CONTENT_SIZE
+ out.reset();
+ } finally {
+ System.setOut(psBackup);
+ cluster.shutdown();
+ }
}
@Test (timeout = 30000)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org