You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/08/26 20:35:05 UTC
[01/52] [abbrv] hadoop git commit: HADOOP-13428. Fixed hadoop-common
build files to generate jdiff by working around a jdiff-bug. Contributed by
Wangda Tan.
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13341 30ee57ceb -> cde3a0052
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99603e90/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index c28a05c..54d1cdd 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -34,6 +34,7 @@
<kdc.resource.dir>src/test/resources/kdc</kdc.resource.dir>
<hadoop.component>common</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
+ <is.hadoop.common.component>true</is.hadoop.common.component>
<wsce.config.dir>../etc/hadoop</wsce.config.dir>
<wsce.config.file>wsce-site.xml</wsce.config.file>
</properties>
@@ -512,6 +513,7 @@
<exclude>src/test/resources/test.har/_masterindex</exclude>
<exclude>src/test/resources/test.har/part-0</exclude>
<exclude>src/test/resources/javakeystoreprovider.password</exclude>
+ <exclude>dev-support/jdiff-workaround.patch</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99603e90/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 0357269..0ee9895 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -160,7 +160,7 @@
<activeByDefault>false</activeByDefault>
</activation>
<properties>
- <jdiff.stable.api>2.6.0</jdiff.stable.api>
+ <jdiff.stable.api>2.7.2</jdiff.stable.api>
<jdiff.stability>-unstable</jdiff.stability>
<!-- Commented out for HADOOP-11776 -->
<!-- Uncomment param name="${jdiff.compatibility}" in javadoc doclet if compatibility is not empty -->
@@ -173,7 +173,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
- <execution>
+ <execution>
<goals>
<goal>javadoc</goal>
</goals>
@@ -241,6 +241,26 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
+
+ <!-- Pre site -->
+ <execution>
+ <id>pre-site</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target if="is.hadoop.common.component">
+ <!-- HADOOP-13428: This is for workaround issues of JDiff
+ - https://sourceforge.net/p/javadiff/bugs/19/ -->
+ <exec executable="patch" input="dev-support/jdiff-workaround.patch" dir="${basedir}">
+ <arg value="-p3"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+
+
<execution>
<id>site</id>
<phase>prepare-package</phase>
@@ -249,7 +269,6 @@
</goals>
<configuration>
<target if="is.hadoop.component">
-
<mkdir dir="${project.build.directory}/docs-src"/>
<!-- Docs -->
@@ -302,6 +321,25 @@
</target>
</configuration>
</execution>
+
+ <!-- post site phase -->
+ <execution>
+ <id>post-site</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target if="is.hadoop.common.component">
+ <!-- HADOOP-13428: Revert temporary patch-->
+ <exec executable="patch" input="dev-support/jdiff-workaround.patch" dir="${basedir}">
+ <arg value="-p3"/>
+ <arg value="-R"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+
</executions>
</plugin>
</plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/52] [abbrv] hadoop git commit: HADOOP-13396. Allow pluggable
audit loggers in KMS. Contributed by Xiao Chen
Posted by aw...@apache.org.
HADOOP-13396. Allow pluggable audit loggers in KMS. Contributed by Xiao Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34761568
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34761568
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34761568
Branch: refs/heads/HADOOP-13341
Commit: 3476156807733505746951f0c9346592742bbbb2
Parents: 6fa9bf4
Author: Xiao Chen <xi...@apache.org>
Authored: Wed Aug 24 10:14:46 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Wed Aug 24 10:14:46 2016 -0700
----------------------------------------------------------------------
.../hadoop-kms/src/main/conf/kms-site.xml | 10 +
.../hadoop/crypto/key/kms/server/KMSAudit.java | 200 ++++++++++---------
.../crypto/key/kms/server/KMSAuditLogger.java | 154 ++++++++++++++
.../crypto/key/kms/server/KMSConfiguration.java | 4 +
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 5 +-
.../key/kms/server/SimpleKMSAuditLogger.java | 96 +++++++++
.../crypto/key/kms/server/TestKMSAudit.java | 43 +++-
.../test/resources/log4j-kmsaudit.properties | 5 +-
8 files changed, 415 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34761568/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
index e8d2344..6c8becf 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
@@ -170,4 +170,14 @@
</description>
</property>
+ <property>
+ <name>hadoop.kms.audit.logger</name>
+ <value>org.apache.hadoop.crypto.key.kms.server.SimpleKMSAuditLogger</value>
+ <description>
+ The audit logger for KMS. It is a comma-separated list of KMSAuditLogger
+ class names. Default is the text-format SimpleKMSAuditLogger only.
+ If this is not configured, default will be used.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34761568/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
index 6a401e0..eef0bce 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java
@@ -17,12 +17,18 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
+import static org.apache.hadoop.crypto.key.kms.server.KMSAuditLogger.AuditEvent;
+import static org.apache.hadoop.crypto.key.kms.server.KMSAuditLogger.OpStatus;
+
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Joiner;
import com.google.common.base.Strings;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@@ -31,67 +37,24 @@ import com.google.common.cache.RemovalNotification;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
+import java.util.HashSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
/**
- * Provides convenience methods for audit logging consistently the different
+ * Provides convenience methods for audit logging consisting different
* types of events.
*/
public class KMSAudit {
-
- private static class AuditEvent {
- private final AtomicLong accessCount = new AtomicLong(-1);
- private final String keyName;
- private final String user;
- private final KMS.KMSOp op;
- private final String extraMsg;
- private final long startTime = System.currentTimeMillis();
-
- private AuditEvent(String keyName, String user, KMS.KMSOp op, String msg) {
- this.keyName = keyName;
- this.user = user;
- this.op = op;
- this.extraMsg = msg;
- }
-
- public String getExtraMsg() {
- return extraMsg;
- }
-
- public AtomicLong getAccessCount() {
- return accessCount;
- }
-
- public String getKeyName() {
- return keyName;
- }
-
- public String getUser() {
- return user;
- }
-
- public KMS.KMSOp getOp() {
- return op;
- }
-
- public long getStartTime() {
- return startTime;
- }
- }
-
- public static enum OpStatus {
- OK, UNAUTHORIZED, UNAUTHENTICATED, ERROR;
- }
-
- private static Set<KMS.KMSOp> AGGREGATE_OPS_WHITELIST = Sets.newHashSet(
+ @VisibleForTesting
+ static final Set<KMS.KMSOp> AGGREGATE_OPS_WHITELIST = Sets.newHashSet(
KMS.KMSOp.GET_KEY_VERSION, KMS.KMSOp.GET_CURRENT_KEY,
KMS.KMSOp.DECRYPT_EEK, KMS.KMSOp.GENERATE_EEK
);
@@ -102,17 +65,21 @@ public class KMSAudit {
public static final String KMS_LOGGER_NAME = "kms-audit";
- private static Logger AUDIT_LOG = LoggerFactory.getLogger(KMS_LOGGER_NAME);
+ private final static Logger LOG = LoggerFactory.getLogger(KMSAudit.class);
+ private final List<KMSAuditLogger> auditLoggers = new LinkedList<>();
/**
* Create a new KMSAudit.
*
- * @param windowMs Duplicate events within the aggregation window are quashed
- * to reduce log traffic. A single message for aggregated
- * events is printed at the end of the window, along with a
- * count of the number of aggregated events.
+ * @param conf The configuration object.
*/
- KMSAudit(long windowMs) {
+ KMSAudit(Configuration conf) {
+ // Duplicate events within the aggregation window are quashed
+ // to reduce log traffic. A single message for aggregated
+ // events is printed at the end of the window, along with a
+ // count of the number of aggregated events.
+ long windowMs = conf.getLong(KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW,
+ KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT);
cache = CacheBuilder.newBuilder()
.expireAfterWrite(windowMs, TimeUnit.MILLISECONDS)
.removalListener(
@@ -122,7 +89,7 @@ public class KMSAudit {
RemovalNotification<String, AuditEvent> entry) {
AuditEvent event = entry.getValue();
if (event.getAccessCount().get() > 0) {
- KMSAudit.this.logEvent(event);
+ KMSAudit.this.logEvent(OpStatus.OK, event);
event.getAccessCount().set(0);
KMSAudit.this.cache.put(entry.getKey(), event);
}
@@ -136,87 +103,127 @@ public class KMSAudit {
cache.cleanUp();
}
}, windowMs / 10, windowMs / 10, TimeUnit.MILLISECONDS);
+ initializeAuditLoggers(conf);
}
- private void logEvent(AuditEvent event) {
- AUDIT_LOG.info(
- "OK[op={}, key={}, user={}, accessCount={}, interval={}ms] {}",
- event.getOp(), event.getKeyName(), event.getUser(),
- event.getAccessCount().get(),
- (System.currentTimeMillis() - event.getStartTime()),
- event.getExtraMsg());
+ /**
+ * Read the KMSAuditLogger classes from configuration. If any loggers fail to
+ * load, a RumTimeException will be thrown.
+ *
+ * @param conf The configuration.
+ * @return Collection of KMSAudigLogger classes.
+ */
+ private Set<Class<? extends KMSAuditLogger>> getAuditLoggerClasses(
+ final Configuration conf) {
+ Set<Class<? extends KMSAuditLogger>> result = new HashSet<>();
+ // getTrimmedStringCollection will remove duplicates.
+ Collection<String> classes =
+ conf.getTrimmedStringCollection(KMSConfiguration.KMS_AUDIT_LOGGER_KEY);
+ if (classes.isEmpty()) {
+ LOG.info("No audit logger configured, using default.");
+ result.add(SimpleKMSAuditLogger.class);
+ return result;
+ }
+
+ for (String c : classes) {
+ try {
+ Class<?> cls = conf.getClassByName(c);
+ result.add(cls.asSubclass(KMSAuditLogger.class));
+ } catch (ClassNotFoundException cnfe) {
+ throw new RuntimeException("Failed to load " + c + ", please check "
+ + "configuration " + KMSConfiguration.KMS_AUDIT_LOGGER_KEY, cnfe);
+ }
+ }
+ return result;
}
- private void op(OpStatus opStatus, final KMS.KMSOp op, final String user,
- final String key, final String extraMsg) {
+ /**
+ * Create a collection of KMSAuditLoggers from configuration, and initialize
+ * them. If any logger failed to be created or initialized, a RunTimeException
+ * is thrown.
+ */
+ private void initializeAuditLoggers(Configuration conf) {
+ Set<Class<? extends KMSAuditLogger>> classes = getAuditLoggerClasses(conf);
+ Preconditions
+ .checkState(!classes.isEmpty(), "Should have at least 1 audit logger.");
+ for (Class<? extends KMSAuditLogger> c : classes) {
+ final KMSAuditLogger logger = ReflectionUtils.newInstance(c, conf);
+ auditLoggers.add(logger);
+ }
+ for (KMSAuditLogger logger: auditLoggers) {
+ try {
+ LOG.info("Initializing audit logger {}", logger.getClass());
+ logger.initialize(conf);
+ } catch (Exception ex) {
+ throw new RuntimeException(
+ "Failed to initialize " + logger.getClass().getName(), ex);
+ }
+ }
+ }
+
+ private void logEvent(final OpStatus status, AuditEvent event) {
+ event.setEndTime(Time.now());
+ for (KMSAuditLogger logger: auditLoggers) {
+ logger.logAuditEvent(status, event);
+ }
+ }
+
+ private void op(final OpStatus opStatus, final KMS.KMSOp op,
+ final UserGroupInformation ugi, final String key, final String remoteHost,
+ final String extraMsg) {
+ final String user = ugi == null ? null: ugi.getShortUserName();
if (!Strings.isNullOrEmpty(user) && !Strings.isNullOrEmpty(key)
&& (op != null)
&& AGGREGATE_OPS_WHITELIST.contains(op)) {
String cacheKey = createCacheKey(user, key, op);
if (opStatus == OpStatus.UNAUTHORIZED) {
cache.invalidate(cacheKey);
- AUDIT_LOG.info("UNAUTHORIZED[op={}, key={}, user={}] {}", op, key, user,
- extraMsg);
+ logEvent(opStatus, new AuditEvent(op, ugi, key, remoteHost, extraMsg));
} else {
try {
AuditEvent event = cache.get(cacheKey, new Callable<AuditEvent>() {
@Override
public AuditEvent call() throws Exception {
- return new AuditEvent(key, user, op, extraMsg);
+ return new AuditEvent(op, ugi, key, remoteHost, extraMsg);
}
});
// Log first access (initialized as -1 so
// incrementAndGet() == 0 implies first access)
if (event.getAccessCount().incrementAndGet() == 0) {
event.getAccessCount().incrementAndGet();
- logEvent(event);
+ logEvent(opStatus, event);
}
} catch (ExecutionException ex) {
throw new RuntimeException(ex);
}
}
} else {
- List<String> kvs = new LinkedList<String>();
- if (op != null) {
- kvs.add("op=" + op);
- }
- if (!Strings.isNullOrEmpty(key)) {
- kvs.add("key=" + key);
- }
- if (!Strings.isNullOrEmpty(user)) {
- kvs.add("user=" + user);
- }
- if (kvs.size() == 0) {
- AUDIT_LOG.info("{} {}", opStatus.toString(), extraMsg);
- } else {
- String join = Joiner.on(", ").join(kvs);
- AUDIT_LOG.info("{}[{}] {}", opStatus.toString(), join, extraMsg);
- }
+ logEvent(opStatus, new AuditEvent(op, ugi, key, remoteHost, extraMsg));
}
}
public void ok(UserGroupInformation user, KMS.KMSOp op, String key,
String extraMsg) {
- op(OpStatus.OK, op, user.getShortUserName(), key, extraMsg);
+ op(OpStatus.OK, op, user, key, "Unknown", extraMsg);
}
public void ok(UserGroupInformation user, KMS.KMSOp op, String extraMsg) {
- op(OpStatus.OK, op, user.getShortUserName(), null, extraMsg);
+ op(OpStatus.OK, op, user, null, "Unknown", extraMsg);
}
public void unauthorized(UserGroupInformation user, KMS.KMSOp op, String key) {
- op(OpStatus.UNAUTHORIZED, op, user.getShortUserName(), key, "");
+ op(OpStatus.UNAUTHORIZED, op, user, key, "Unknown", "");
}
public void error(UserGroupInformation user, String method, String url,
String extraMsg) {
- op(OpStatus.ERROR, null, user.getShortUserName(), null, "Method:'" + method
+ op(OpStatus.ERROR, null, user, null, "Unknown", "Method:'" + method
+ "' Exception:'" + extraMsg + "'");
}
public void unauthenticated(String remoteHost, String method,
String url, String extraMsg) {
- op(OpStatus.UNAUTHENTICATED, null, null, null, "RemoteHost:"
+ op(OpStatus.UNAUTHENTICATED, null, null, null, remoteHost, "RemoteHost:"
+ remoteHost + " Method:" + method
+ " URL:" + url + " ErrorMsg:'" + extraMsg + "'");
}
@@ -227,6 +234,13 @@ public class KMSAudit {
public void shutdown() {
executor.shutdownNow();
+ for (KMSAuditLogger logger : auditLoggers) {
+ try {
+ logger.cleanup();
+ } catch (Exception ex) {
+ LOG.error("Failed to cleanup logger {}", logger.getClass(), ex);
+ }
+ }
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34761568/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java
new file mode 100644
index 0000000..a539724
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuditLogger.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Interface defining a KMS audit logger.
+ * <p>
+ * IMPORTANT WARNING: Audit logs should be strictly backwards-compatible,
+ * because there are usually parsing tools highly dependent on the audit log
+ * formatting. Different tools have different ways of parsing the audit log, so
+ * changing the audit log output in any way is considered incompatible,
+ * and will haunt the consumer tools / developers. Don't do it.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+interface KMSAuditLogger {
+ enum OpStatus {
+ OK, UNAUTHORIZED, UNAUTHENTICATED, ERROR;
+ }
+
+ /**
+ * Class defining an audit event.
+ */
+ class AuditEvent {
+ private final AtomicLong accessCount = new AtomicLong(-1);
+ private final KMS.KMSOp op;
+ private final String keyName;
+ private final String user;
+ private final String impersonator;
+ private final String remoteHost;
+ private final String extraMsg;
+ private final long startTime = System.currentTimeMillis();
+ private long endTime = startTime;
+
+ AuditEvent(KMS.KMSOp op, UserGroupInformation ugi, String keyName,
+ String remoteHost, String msg) {
+ this.keyName = keyName;
+ if (ugi == null) {
+ this.user = null;
+ this.impersonator = null;
+ } else {
+ this.user = ugi.getShortUserName();
+ if (ugi.getAuthenticationMethod()
+ == UserGroupInformation.AuthenticationMethod.PROXY) {
+ this.impersonator = ugi.getRealUser().getUserName();
+ } else {
+ this.impersonator = null;
+ }
+ }
+ this.remoteHost = remoteHost;
+ this.op = op;
+ this.extraMsg = msg;
+ }
+
+ public AtomicLong getAccessCount() {
+ return accessCount;
+ }
+
+ public KMS.KMSOp getOp() {
+ return op;
+ }
+
+ public String getKeyName() {
+ return keyName;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public String getImpersonator() {
+ return impersonator;
+ }
+
+ public String getRemoteHost() {
+ return remoteHost;
+ }
+
+ public String getExtraMsg() {
+ return extraMsg;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getEndTime() {
+ return endTime;
+ }
+
+ /**
+ * Set the time this audit event is finished.
+ */
+ void setEndTime(long endTime) {
+ this.endTime = endTime;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("op=" + op).append(", keyName=" + keyName)
+ .append(", user=" + user).append(", impersonator=" + impersonator)
+ .append(", remoteHost=" + remoteHost)
+ .append(", extraMsg=" + extraMsg);
+ return sb.toString();
+ }
+ }
+
+ /**
+ * Clean up the audit logger.
+ *
+ * @throws IOException
+ */
+ void cleanup() throws IOException;
+
+ /**
+ * Initialize the audit logger.
+ *
+ * @param conf The configuration object.
+ * @throws IOException
+ */
+ void initialize(Configuration conf) throws IOException;
+
+ /**
+ * Log an audit event.
+ *
+ * @param status The status of the event.
+ * @param event The audit event.
+ */
+ void logAuditEvent(final OpStatus status, final AuditEvent event);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34761568/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index 23c983f..600f1e9 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -58,6 +58,10 @@ public class KMSConfiguration {
// Delay for Audit logs that need aggregation
public static final String KMS_AUDIT_AGGREGATION_WINDOW = CONFIG_PREFIX +
"audit.aggregation.window.ms";
+
+ // KMS Audit logger classes to use
+ public static final String KMS_AUDIT_LOGGER_KEY = CONFIG_PREFIX +
+ "audit.logger";
public static final boolean KEY_CACHE_ENABLE_DEFAULT = true;
// 10 mins
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34761568/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index 3b439a4..e3d1a93 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -147,10 +147,7 @@ public class KMSWebApp implements ServletContextListener {
unauthenticatedCallsMeter = metricRegistry.register(
UNAUTHENTICATED_CALLS_METER, new Meter());
- kmsAudit =
- new KMSAudit(kmsConf.getLong(
- KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW,
- KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT));
+ kmsAudit = new KMSAudit(kmsConf);
// this is required for the the JMXJsonServlet to work properly.
// the JMXJsonServlet is behind the authentication filter,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34761568/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/SimpleKMSAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/SimpleKMSAuditLogger.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/SimpleKMSAuditLogger.java
new file mode 100644
index 0000000..4dcbe2c
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/SimpleKMSAuditLogger.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import static org.apache.hadoop.crypto.key.kms.server.KMSAudit.KMS_LOGGER_NAME;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A simple text format audit logger. This is the default.
+ * <p>
+ * IMPORTANT WARNING: Audit logs should be strictly backwards-compatible,
+ * because there are usually parsing tools highly dependent on the audit log
+ * formatting. Different tools have different ways of parsing the audit log, so
+ * changing the audit log output in any way is considered incompatible,
+ * and will haunt the consumer tools / developers. Don't do it.
+ */
+class SimpleKMSAuditLogger implements KMSAuditLogger {
+ final private Logger auditLog = LoggerFactory.getLogger(KMS_LOGGER_NAME);
+
+ @Override
+ public void cleanup() throws IOException {
+ }
+
+ @Override
+ public void initialize(Configuration conf) throws IOException {
+ }
+
+ @Override
+ public void logAuditEvent(final OpStatus status, final AuditEvent event) {
+ if (!Strings.isNullOrEmpty(event.getUser()) && !Strings
+ .isNullOrEmpty(event.getKeyName()) && (event.getOp() != null)
+ && KMSAudit.AGGREGATE_OPS_WHITELIST.contains(event.getOp())) {
+ switch (status) {
+ case OK:
+ auditLog.info(
+ "{}[op={}, key={}, user={}, accessCount={}, interval={}ms] {}",
+ status, event.getOp(), event.getKeyName(), event.getUser(),
+ event.getAccessCount().get(),
+ (event.getEndTime() - event.getStartTime()), event.getExtraMsg());
+ break;
+ case UNAUTHORIZED:
+ logAuditSimpleFormat(status, event);
+ break;
+ default:
+ logAuditSimpleFormat(status, event);
+ break;
+ }
+ } else {
+ logAuditSimpleFormat(status, event);
+ }
+ }
+
+ private void logAuditSimpleFormat(final OpStatus status,
+ final AuditEvent event) {
+ final List<String> kvs = new LinkedList<>();
+ if (event.getOp() != null) {
+ kvs.add("op=" + event.getOp());
+ }
+ if (!Strings.isNullOrEmpty(event.getKeyName())) {
+ kvs.add("key=" + event.getKeyName());
+ }
+ if (!Strings.isNullOrEmpty(event.getUser())) {
+ kvs.add("user=" + event.getUser());
+ }
+ if (kvs.isEmpty()) {
+ auditLog.info("{} {}", status, event.getExtraMsg());
+ } else {
+ final String join = Joiner.on(", ").join(kvs);
+ auditLog.info("{}[{}] {}", status, join, event.getExtraMsg());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34761568/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
index 53948ee..2d9443f 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
@@ -23,10 +23,13 @@ import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.kms.server.KMS.KMSOp;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ThreadUtil;
import org.apache.log4j.LogManager;
import org.apache.log4j.PropertyConfigurator;
@@ -37,6 +40,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
public class TestKMSAudit {
@@ -45,7 +49,7 @@ public class TestKMSAudit {
private FilterOut filterOut;
private PrintStream capturedOut;
- private KMSAudit kmsAudit;
+ private KMSAudit kmsAudit;
private static class FilterOut extends FilterOutputStream {
public FilterOut(OutputStream out) {
@@ -71,9 +75,8 @@ public class TestKMSAudit {
ThreadUtil.getResourceAsStream("log4j-kmsaudit.properties");
PropertyConfigurator.configure(is);
IOUtils.closeStream(is);
-
- this.kmsAudit =
- new KMSAudit(KMSConfiguration.KMS_AUDIT_AGGREGATION_WINDOW_DEFAULT);
+ Configuration conf = new Configuration();
+ this.kmsAudit = new KMSAudit(conf);
}
@After
@@ -175,4 +178,36 @@ public class TestKMSAudit {
+ "ERROR\\[user=luser\\] Method:'method' Exception:'testmsg'"
+ "UNAUTHENTICATED RemoteHost:remotehost Method:method URL:url ErrorMsg:'testmsg'"));
}
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testInitAuditLoggers() throws Exception {
+ // Default should be the simple logger
+ List<KMSAuditLogger> loggers = (List<KMSAuditLogger>) Whitebox
+ .getInternalState(kmsAudit, "auditLoggers");
+ Assert.assertEquals(1, loggers.size());
+ Assert.assertEquals(SimpleKMSAuditLogger.class, loggers.get(0).getClass());
+
+ // Explicitly configure the simple logger. Duplicates are ignored.
+ final Configuration conf = new Configuration();
+ conf.set(KMSConfiguration.KMS_AUDIT_LOGGER_KEY,
+ SimpleKMSAuditLogger.class.getName() + ", "
+ + SimpleKMSAuditLogger.class.getName());
+ final KMSAudit audit = new KMSAudit(conf);
+ loggers =
+ (List<KMSAuditLogger>) Whitebox.getInternalState(audit, "auditLoggers");
+ Assert.assertEquals(1, loggers.size());
+ Assert.assertEquals(SimpleKMSAuditLogger.class, loggers.get(0).getClass());
+
+ // If any loggers unable to load, init should fail.
+ conf.set(KMSConfiguration.KMS_AUDIT_LOGGER_KEY,
+ SimpleKMSAuditLogger.class.getName() + ",unknown");
+ try {
+ new KMSAudit(conf);
+ Assert.fail("loggers configured but invalid, init should fail.");
+ } catch (Exception ex) {
+ GenericTestUtils
+ .assertExceptionContains(KMSConfiguration.KMS_AUDIT_LOGGER_KEY, ex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34761568/hadoop-common-project/hadoop-kms/src/test/resources/log4j-kmsaudit.properties
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/resources/log4j-kmsaudit.properties b/hadoop-common-project/hadoop-kms/src/test/resources/log4j-kmsaudit.properties
index cca6941..b8a4013 100644
--- a/hadoop-common-project/hadoop-kms/src/test/resources/log4j-kmsaudit.properties
+++ b/hadoop-common-project/hadoop-kms/src/test/resources/log4j-kmsaudit.properties
@@ -16,10 +16,13 @@
# limitations under the License.
#
+log4j.rootLogger=INFO, kms-audit
+
# LOG Appender
log4j.appender.kms-audit=org.apache.log4j.ConsoleAppender
log4j.appender.kms-audit.Target=System.err
log4j.appender.kms-audit.layout=org.apache.log4j.PatternLayout
log4j.appender.kms-audit.layout.ConversionPattern=%m
-log4j.rootLogger=INFO, kms-audit
\ No newline at end of file
+# disable default logging in KMSAudit class
+log4j.logger.org.apache.hadoop.crypto.key.kms.server.KMSAudit=OFF
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/52] [abbrv] hadoop git commit: HADOOP-13526. Add detailed logging
in KMS for the authentication failure of proxy user. Contributed by Suraj
Acharya.
Posted by aw...@apache.org.
HADOOP-13526. Add detailed logging in KMS for the authentication failure of proxy user. Contributed by Suraj Acharya.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4070caad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4070caad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4070caad
Branch: refs/heads/HADOOP-13341
Commit: 4070caad70db49b50554088d29ac2fbc7ba62a0a
Parents: 3ca4d6d
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Aug 22 18:06:53 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Aug 22 18:09:35 2016 -0700
----------------------------------------------------------------------
.../web/DelegationTokenAuthenticationFilter.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4070caad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
index fb6817e..112c952 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationFilter.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.util.HttpExceptionUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
@@ -81,6 +83,9 @@ public class DelegationTokenAuthenticationFilter
private static final String ERROR_EXCEPTION_JSON = "exception";
private static final String ERROR_MESSAGE_JSON = "message";
+ private static final Logger LOG = LoggerFactory.getLogger(
+ DelegationTokenAuthenticationFilter.class);
+
/**
* Sets an external <code>DelegationTokenSecretManager</code> instance to
* manage creation and verification of Delegation Tokens.
@@ -261,6 +266,11 @@ public class DelegationTokenAuthenticationFilter
HttpExceptionUtils.createServletExceptionResponse(response,
HttpServletResponse.SC_FORBIDDEN, ex);
requestCompleted = true;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Authentication exception: " + ex.getMessage(), ex);
+ } else {
+ LOG.warn("Authentication exception: " + ex.getMessage());
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/52] [abbrv] hadoop git commit: YARN-4491. yarn list command to
support filtering by tags. Contributed by Varun Saxena
Posted by aw...@apache.org.
YARN-4491. yarn list command to support filtering by tags. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/143c59e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/143c59e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/143c59e4
Branch: refs/heads/HADOOP-13341
Commit: 143c59e4c5a811eb2c12cf6626d558f9b8796e03
Parents: 8aae8d6
Author: Naganarasimha <na...@apache.org>
Authored: Wed Aug 24 01:53:02 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Wed Aug 24 01:53:02 2016 +0530
----------------------------------------------------------------------
.../hadoop/mapred/ResourceMgrDelegate.java | 10 +
.../hadoop/yarn/client/api/YarnClient.java | 25 ++
.../yarn/client/api/impl/YarnClientImpl.java | 11 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 42 +++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 241 ++++++++++++++++---
5 files changed, 282 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index cc164fd..159b518 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -349,6 +349,16 @@ public class ResourceMgrDelegate extends YarnClient {
}
@Override
+ public List<ApplicationReport> getApplications(
+ Set<String> applicationTypes,
+ EnumSet<YarnApplicationState> applicationStates,
+ Set<String> applicationTags)
+ throws YarnException, IOException {
+ return client.getApplications(
+ applicationTypes, applicationStates, applicationTags);
+ }
+
+ @Override
public List<ApplicationReport> getApplications(Set<String> queues,
Set<String> users, Set<String> applicationTypes,
EnumSet<YarnApplicationState> applicationStates) throws YarnException,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 218bb34..619ea0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -314,6 +314,31 @@ public abstract class YarnClient extends AbstractService {
/**
* <p>
+ * Get a report (ApplicationReport) of Applications matching the given
+ * application types, application states and application tags in the cluster.
+ * </p>
+ *
+ * <p>
+ * If the user does not have <code>VIEW_APP</code> access for an application
+ * then the corresponding report will be filtered as described in
+ * {@link #getApplicationReport(ApplicationId)}.
+ * </p>
+ *
+ * @param applicationTypes set of application types you are interested in
+ * @param applicationStates set of application states you are interested in
+ * @param applicationTags set of application tags you are interested in
+ * @return a list of reports of applications
+ * @throws YarnException
+ * @throws IOException
+ */
+ public abstract List<ApplicationReport> getApplications(
+ Set<String> applicationTypes,
+ EnumSet<YarnApplicationState> applicationStates,
+ Set<String> applicationTags) throws YarnException,
+ IOException;
+
+ /**
+ * <p>
* Get a report (ApplicationReport) of Applications matching the given users,
* queues, application types and application states in the cluster. If any of
* the params is set to null, it is not used when filtering.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index dee31c5..b91fde0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -505,6 +505,17 @@ public class YarnClientImpl extends YarnClient {
}
@Override
+ public List<ApplicationReport> getApplications(Set<String> applicationTypes,
+ EnumSet<YarnApplicationState> applicationStates,
+ Set<String> applicationTags) throws YarnException, IOException {
+ GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
+ request.setApplicationTags(applicationTags);
+ GetApplicationsResponse response = rmClient.getApplications(request);
+ return response.getApplicationList();
+ }
+
+ @Override
public List<ApplicationReport> getApplications(Set<String> queues,
Set<String> users, Set<String> applicationTypes,
EnumSet<YarnApplicationState> applicationStates) throws YarnException,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 865ce00..f0b1c47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -73,6 +73,7 @@ public class ApplicationCLI extends YarnCLI {
private static final String APP_TYPE_CMD = "appTypes";
private static final String APP_STATE_CMD = "appStates";
+ private static final String APP_TAG_CMD = "appTags";
private static final String ALLSTATES_OPTION = "ALL";
private static final String QUEUE_CMD = "queue";
public static final String APPLICATION = "application";
@@ -102,8 +103,9 @@ public class ApplicationCLI extends YarnCLI {
"Prints the status of the application.");
opts.addOption(LIST_CMD, false, "List applications. "
+ "Supports optional use of -appTypes to filter applications "
- + "based on application type, "
- + "and -appStates to filter applications based on application state.");
+ + "based on application type, -appStates to filter applications "
+ + "based on application state and -appTags to filter applications "
+ + "based on application tag.");
opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a "
+ "different queue.");
opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to"
@@ -123,6 +125,13 @@ public class ApplicationCLI extends YarnCLI {
appStateOpt.setArgs(Option.UNLIMITED_VALUES);
appStateOpt.setArgName("States");
opts.addOption(appStateOpt);
+ Option appTagOpt = new Option(APP_TAG_CMD, true, "Works with -list to "
+ + "filter applications based on input comma-separated list of "
+ + "application tags.");
+ appTagOpt.setValueSeparator(',');
+ appTagOpt.setArgs(Option.UNLIMITED_VALUES);
+ appTagOpt.setArgName("Tags");
+ opts.addOption(appTagOpt);
opts.addOption(APP_ID, true, "Specify Application Id to be operated");
opts.addOption(UPDATE_PRIORITY, true,
"update priority of an application. ApplicationId can be"
@@ -229,7 +238,19 @@ public class ApplicationCLI extends YarnCLI {
}
}
}
- listApplications(appTypes, appStates);
+
+ Set<String> appTags = new HashSet<String>();
+ if (cliParser.hasOption(APP_TAG_CMD)) {
+ String[] tags = cliParser.getOptionValues(APP_TAG_CMD);
+ if (tags != null) {
+ for (String tag : tags) {
+ if (!tag.trim().isEmpty()) {
+ appTags.add(tag.trim());
+ }
+ }
+ }
+ }
+ listApplications(appTypes, appStates, appTags);
} else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
if (args.length != 3) {
printUsage(title, opts);
@@ -434,17 +455,18 @@ public class ApplicationCLI extends YarnCLI {
}
/**
- * Lists the applications matching the given application Types And application
- * States present in the Resource Manager
+ * Lists the applications matching the given application Types, application
+ * States and application Tags present in the Resource Manager.
*
* @param appTypes
* @param appStates
+ * @param appTags
* @throws YarnException
* @throws IOException
*/
private void listApplications(Set<String> appTypes,
- EnumSet<YarnApplicationState> appStates) throws YarnException,
- IOException {
+ EnumSet<YarnApplicationState> appStates, Set<String> appTags)
+ throws YarnException, IOException {
PrintWriter writer = new PrintWriter(
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
if (allAppStates) {
@@ -460,11 +482,11 @@ public class ApplicationCLI extends YarnCLI {
}
List<ApplicationReport> appsReport = client.getApplications(appTypes,
- appStates);
+ appStates, appTags);
writer.println("Total number of applications (application-types: "
- + appTypes + " and states: " + appStates + ")" + ":"
- + appsReport.size());
+ + appTypes + ", states: " + appStates + " and tags: " + appTags + ")"
+ + ":" + appsReport.size());
writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name",
"Application-Type", "User", "Queue", "State", "Final-State",
"Progress", "Tracking-URL");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/143c59e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 6d7aaa7..1cd513b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -87,6 +87,7 @@ import org.junit.Test;
import org.mortbay.log.Log;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
public class TestYarnCLI {
@@ -373,8 +374,10 @@ public class TestYarnCLI {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
- List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ Sets.newHashSet("tag1", "tag3"), false, Priority.UNDEFINED, "", "");
+ List<ApplicationReport> applicationReports =
+ new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
@@ -383,7 +386,8 @@ public class TestYarnCLI {
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
- null);
+ null, Sets.newHashSet("tag2", "tag3"), false, Priority.UNDEFINED,
+ "", "");
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -392,7 +396,8 @@ public class TestYarnCLI {
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
- null);
+ null, Sets.newHashSet("tag1", "tag4"), false, Priority.UNDEFINED,
+ "", "");
applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -400,8 +405,9 @@ public class TestYarnCLI {
applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4),
"user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
- null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
+ "NON-MAPREDUCE", null, Sets.newHashSet("tag1"), false,
+ Priority.UNDEFINED, "", "");
applicationReports.add(newApplicationReport4);
ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
@@ -409,8 +415,8 @@ public class TestYarnCLI {
applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5),
"user5", "queue5", "appname5", "host5", 128, null,
YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5,
- FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE",
- null);
+ FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", null,
+ Sets.newHashSet("tag2", "tag4"), false, Priority.UNDEFINED, "", "");
applicationReports.add(newApplicationReport5);
ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
@@ -419,7 +425,7 @@ public class TestYarnCLI {
"user6", "queue6", "appname6", "host6", 129, null,
YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6,
FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG",
- null);
+ null, new HashSet<String>(), false, Priority.UNDEFINED, "", "");
applicationReports.add(newApplicationReport6);
// Test command yarn application -list
@@ -433,16 +439,18 @@ public class TestYarnCLI {
appState1.add(YarnApplicationState.RUNNING);
appState1.add(YarnApplicationState.ACCEPTED);
appState1.add(YarnApplicationState.SUBMITTED);
- when(client.getApplications(appType1, appState1)).thenReturn(
- getApplicationReports(applicationReports, appType1, appState1, false));
+ Set<String> appTag = new HashSet<String>();
+ when(client.getApplications(appType1, appState1, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType1, appState1, appTag, false));
int result = cli.run(new String[] { "application", "-list" });
assertEquals(0, result);
- verify(client).getApplications(appType1, appState1);
+ verify(client).getApplications(appType1, appState1, appTag);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType1
- + " and states: " + appState1 + ")" + ":" + 4);
+ + ", states: " + appState1 + " and tags: " + appTag + ")" + ":" + 4);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -487,17 +495,18 @@ public class TestYarnCLI {
appState2.add(YarnApplicationState.RUNNING);
appState2.add(YarnApplicationState.ACCEPTED);
appState2.add(YarnApplicationState.SUBMITTED);
- when(client.getApplications(appType2, appState2)).thenReturn(
- getApplicationReports(applicationReports, appType2, appState2, false));
+ when(client.getApplications(appType2, appState2, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType2, appState2, appTag, false));
result =
cli.run(new String[] { "application", "-list", "-appTypes",
"YARN, ,, NON-YARN", " ,, ,," });
assertEquals(0, result);
- verify(client).getApplications(appType2, appState2);
+ verify(client).getApplications(appType2, appState2, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType2
- + " and states: " + appState2 + ")" + ":" + 1);
+ + ", states: " + appState2 + " and tags: " + appTag + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -522,17 +531,18 @@ public class TestYarnCLI {
appState3.add(YarnApplicationState.FINISHED);
appState3.add(YarnApplicationState.FAILED);
- when(client.getApplications(appType3, appState3)).thenReturn(
- getApplicationReports(applicationReports, appType3, appState3, false));
+ when(client.getApplications(appType3, appState3, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType3, appState3, appTag, false));
result =
cli.run(new String[] { "application", "-list", "--appStates",
"FINISHED ,, , FAILED", ",,FINISHED" });
assertEquals(0, result);
- verify(client).getApplications(appType3, appState3);
+ verify(client).getApplications(appType3, appState3, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType3
- + " and states: " + appState3 + ")" + ":" + 2);
+ + ", states: " + appState3 + " and tags: " + appTag + ")" + ":" + 2);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -565,17 +575,18 @@ public class TestYarnCLI {
appState4.add(YarnApplicationState.FINISHED);
appState4.add(YarnApplicationState.FAILED);
- when(client.getApplications(appType4, appState4)).thenReturn(
- getApplicationReports(applicationReports, appType4, appState4, false));
+ when(client.getApplications(appType4, appState4, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType4, appState4, appTag, false));
result =
cli.run(new String[] { "application", "-list", "--appTypes",
"YARN,NON-YARN", "--appStates", "FINISHED ,, , FAILED" });
assertEquals(0, result);
- verify(client).getApplications(appType2, appState2);
+ verify(client).getApplications(appType2, appState2, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType4
- + " and states: " + appState4 + ")" + ":" + 1);
+ + ", states: " + appState4 + " and tags: " + appTag + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -620,17 +631,18 @@ public class TestYarnCLI {
EnumSet<YarnApplicationState> appState5 =
EnumSet.noneOf(YarnApplicationState.class);
appState5.add(YarnApplicationState.FINISHED);
- when(client.getApplications(appType5, appState5)).thenReturn(
- getApplicationReports(applicationReports, appType5, appState5, true));
+ when(client.getApplications(appType5, appState5, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType5, appState5, appTag, true));
result =
cli.run(new String[] { "application", "-list", "--appStates",
"FINISHED ,, , ALL" });
assertEquals(0, result);
- verify(client).getApplications(appType5, appState5);
+ verify(client).getApplications(appType5, appState5, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType5
- + " and states: " + appState5 + ")" + ":" + 6);
+ + ", states: " + appState5 + " and tags: " + appTag + ")" + ":" + 6);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -680,17 +692,18 @@ public class TestYarnCLI {
EnumSet<YarnApplicationState> appState6 =
EnumSet.noneOf(YarnApplicationState.class);
appState6.add(YarnApplicationState.FINISHED);
- when(client.getApplications(appType6, appState6)).thenReturn(
- getApplicationReports(applicationReports, appType6, appState6, false));
+ when(client.getApplications(appType6, appState6, appTag)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType6, appState6, appTag, false));
result =
cli.run(new String[] { "application", "-list", "-appTypes",
"YARN, ,, NON-YARN", "--appStates", "finished" });
assertEquals(0, result);
- verify(client).getApplications(appType6, appState6);
+ verify(client).getApplications(appType6, appState6, appTag);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType6
- + " and states: " + appState6 + ")" + ":" + 1);
+ + ", states: " + appState6 + " and tags: " + appTag + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -705,12 +718,146 @@ public class TestYarnCLI {
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
+
+ // Test command yarn application with tags.
+ sysOutStream.reset();
+ Set<String> appTag1 = Sets.newHashSet("tag1");
+ when(client.getApplications(appType1, appState1, appTag1)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType1, appState1, appTag1, false));
+ result =
+ cli.run(new String[] { "application", "-list", "-appTags", "tag1" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType1, appState1, appTag1);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType1
+ + ", states: " + appState1 + " and tags: " + appTag1 + ")" + ":" + 2);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0005\t ");
+ pw.print("appname\t YARN\t user\t ");
+ pw.print("queue\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 53.79%");
+ pw.println("\t N/A");
+ pw.print(" application_1234_0007\t ");
+ pw.print("appname3\t MAPREDUCE\t user3\t ");
+ pw.print("queue3\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 73.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ EnumSet<YarnApplicationState> appState7 =
+ EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.FAILED);
+ when(client.getApplications(appType1, appState7, appTag1)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType1, appState7, appTag1, false));
+ result = cli.run(
+ new String[] { "application", "-list", "-appStates", "RUNNING,FAILED",
+ "-appTags", "tag1" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType1, appState7, appTag1);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType1
+ + ", states: " + appState7 + " and tags: " + appTag1 + ")" + ":" + 3);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0005\t ");
+ pw.print("appname\t YARN\t user\t ");
+ pw.print("queue\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 53.79%");
+ pw.println("\t N/A");
+ pw.print(" application_1234_0007\t ");
+ pw.print("appname3\t MAPREDUCE\t user3\t ");
+ pw.print("queue3\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 73.79%");
+ pw.println("\t N/A");
+ pw.print(" application_1234_0008\t ");
+ pw.print("appname4\t NON-MAPREDUCE\t user4\t ");
+ pw.print("queue4\t FAILED\t ");
+ pw.print("SUCCEEDED\t 83.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ Set<String> appType9 = Sets.newHashSet("YARN");
+ Set<String> appTag2 = Sets.newHashSet("tag3");
+ when(client.getApplications(appType9, appState1, appTag2)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType9, appState1, appTag2, false));
+ result = cli.run(new String[] { "application", "-list", "-appTypes", "YARN",
+ "-appTags", "tag3" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType9, appState1, appTag2);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType9
+ + ", states: " + appState1 + " and tags: " + appTag2 + ")" + ":" + 1);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0005\t ");
+ pw.print("appname\t YARN\t user\t ");
+ pw.print("queue\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 53.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ Set<String> appType10 = Sets.newHashSet("HIVE");
+ Set<String> appTag3 = Sets.newHashSet("tag4");
+ EnumSet<YarnApplicationState> appState10 =
+ EnumSet.of(YarnApplicationState.ACCEPTED);
+ when(client.getApplications(appType10, appState10, appTag3)).thenReturn(
+ getApplicationReports(
+ applicationReports, appType10, appState10, appTag3, false));
+ result = cli.run(new String[] { "application", "-list", "-appTypes", "HIVE",
+ "-appStates", "ACCEPTED", "-appTags", "tag4" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType10, appState10, appTag3);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType10
+ + ", states: " + appState10 + " and tags: " + appTag3 + ")" + ":" + 1);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0009\t ");
+ pw.print("appname5\t HIVE\t user5\t ");
+ pw.print("queue5\t ACCEPTED\t ");
+ pw.print("KILLED\t 93.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(10)).write(any(byte[].class), anyInt(), anyInt());
}
private List<ApplicationReport> getApplicationReports(
List<ApplicationReport> applicationReports,
Set<String> appTypes, EnumSet<YarnApplicationState> appStates,
- boolean allStates) {
+ Set<String> appTags, boolean allStates) {
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
@@ -732,6 +879,22 @@ public class TestYarnCLI {
}
}
+ if (appTags != null && !appTags.isEmpty()) {
+ Set<String> tags = appReport.getApplicationTags();
+ if (tags == null || tags.isEmpty()) {
+ continue;
+ }
+ boolean match = false;
+ for (String appTag : appTags) {
+ if (tags.contains(appTag)) {
+ match = true;
+ break;
+ }
+ }
+ if (!match) {
+ continue;
+ }
+ }
appReports.add(appReport);
}
return appReports;
@@ -1816,6 +1979,9 @@ public class TestYarnCLI {
pw.println(" state can be one of the following:");
pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN");
pw.println(" NING,FINISHED,FAILED,KILLED");
+ pw.println(" -appTags <Tags> Works with -list to filter applications");
+ pw.println(" based on input comma-separated list of");
+ pw.println(" application tags.");
pw.println(" -appTypes <Types> Works with -list to filter applications");
pw.println(" based on input comma-separated list of");
pw.println(" application types.");
@@ -1825,9 +1991,10 @@ public class TestYarnCLI {
pw.println(" with space");
pw.println(" -list List applications. Supports optional use");
pw.println(" of -appTypes to filter applications based");
- pw.println(" on application type, and -appStates to");
- pw.println(" filter applications based on application");
- pw.println(" state.");
+ pw.println(" on application type, -appStates to filter");
+ pw.println(" applications based on application state");
+ pw.println(" and -appTags to filter applications based");
+ pw.println(" on application tag.");
pw.println(" -movetoqueue <Application ID> Moves the application to a different");
pw.println(" queue.");
pw.println(" -queue <Queue Name> Works with the movetoqueue command to");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/52] [abbrv] hadoop git commit: HDFS-8915.
TestFSNamesystem.testFSLockGetWaiterCount fails intermittently. Contributed
by Masatake Iwasaki.
Posted by aw...@apache.org.
HDFS-8915. TestFSNamesystem.testFSLockGetWaiterCount fails intermittently. Contributed by Masatake Iwasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13fb1b50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13fb1b50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13fb1b50
Branch: refs/heads/HADOOP-13341
Commit: 13fb1b50e608558b2970184908ee5b9fcd7eb7b6
Parents: 46e02ab
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Aug 26 09:51:32 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Aug 26 09:53:10 2016 -0500
----------------------------------------------------------------------
.../hdfs/server/namenode/TestFSNamesystem.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fb1b50/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
index b9a2d15..572b40d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
@@ -30,6 +30,7 @@ import java.net.InetAddress;
import java.net.URI;
import java.util.Collection;
+import com.google.common.base.Supplier;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
@@ -56,6 +57,7 @@ import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeoutException;
public class TestFSNamesystem {
@@ -271,9 +273,16 @@ public class TestFSNamesystem {
}
latch.await();
- Thread.sleep(10); // Lets all threads get BLOCKED
- Assert.assertEquals("Expected number of blocked thread not found",
- threadCount, rwLock.getQueueLength());
+ try {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return (threadCount == rwLock.getQueueLength());
+ }
+ }, 10, 1000);
+ } catch (TimeoutException e) {
+ fail("Expected number of blocked thread not found");
+ }
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/52] [abbrv] hadoop git commit: HDFS-8905. Refactor
DFSInputStream#ReaderStrategy. Contributed by Kai Zheng and Sammi Chen
Posted by aw...@apache.org.
HDFS-8905. Refactor DFSInputStream#ReaderStrategy. Contributed by Kai Zheng and Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/793447f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/793447f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/793447f7
Branch: refs/heads/HADOOP-13341
Commit: 793447f79924c97c2b562d5e41fa85adf19673fe
Parents: ec252ce
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Aug 24 21:57:23 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Aug 24 21:57:23 2016 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSInputStream.java | 235 ++-----------------
.../hadoop/hdfs/DFSStripedInputStream.java | 23 +-
.../org/apache/hadoop/hdfs/ReadStatistics.java | 106 +++++++++
.../org/apache/hadoop/hdfs/ReaderStrategy.java | 215 +++++++++++++++++
.../hadoop/hdfs/client/HdfsDataInputStream.java | 3 +-
.../apache/hadoop/hdfs/util/IOUtilsClient.java | 16 ++
.../hadoop/hdfs/TestExternalBlockReader.java | 4 +-
7 files changed, 374 insertions(+), 228 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/793447f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 6132f83..7a10ba4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.hdfs.server.datanode.ReplicaNotFoundException;
import org.apache.hadoop.hdfs.shortcircuit.ClientMmap;
+import org.apache.hadoop.hdfs.util.IOUtilsClient;
import org.apache.hadoop.io.ByteBufferPool;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
@@ -145,94 +146,6 @@ public class DFSInputStream extends FSInputStream
return extendedReadBuffers;
}
- public static class ReadStatistics {
- public ReadStatistics() {
- clear();
- }
-
- public ReadStatistics(ReadStatistics rhs) {
- this.totalBytesRead = rhs.getTotalBytesRead();
- this.totalLocalBytesRead = rhs.getTotalLocalBytesRead();
- this.totalShortCircuitBytesRead = rhs.getTotalShortCircuitBytesRead();
- this.totalZeroCopyBytesRead = rhs.getTotalZeroCopyBytesRead();
- }
-
- /**
- * @return The total bytes read. This will always be at least as
- * high as the other numbers, since it includes all of them.
- */
- public long getTotalBytesRead() {
- return totalBytesRead;
- }
-
- /**
- * @return The total local bytes read. This will always be at least
- * as high as totalShortCircuitBytesRead, since all short-circuit
- * reads are also local.
- */
- public long getTotalLocalBytesRead() {
- return totalLocalBytesRead;
- }
-
- /**
- * @return The total short-circuit local bytes read.
- */
- public long getTotalShortCircuitBytesRead() {
- return totalShortCircuitBytesRead;
- }
-
- /**
- * @return The total number of zero-copy bytes read.
- */
- public long getTotalZeroCopyBytesRead() {
- return totalZeroCopyBytesRead;
- }
-
- /**
- * @return The total number of bytes read which were not local.
- */
- public long getRemoteBytesRead() {
- return totalBytesRead - totalLocalBytesRead;
- }
-
- void addRemoteBytes(long amt) {
- this.totalBytesRead += amt;
- }
-
- void addLocalBytes(long amt) {
- this.totalBytesRead += amt;
- this.totalLocalBytesRead += amt;
- }
-
- void addShortCircuitBytes(long amt) {
- this.totalBytesRead += amt;
- this.totalLocalBytesRead += amt;
- this.totalShortCircuitBytesRead += amt;
- }
-
- void addZeroCopyBytes(long amt) {
- this.totalBytesRead += amt;
- this.totalLocalBytesRead += amt;
- this.totalShortCircuitBytesRead += amt;
- this.totalZeroCopyBytesRead += amt;
- }
-
- void clear() {
- this.totalBytesRead = 0;
- this.totalLocalBytesRead = 0;
- this.totalShortCircuitBytesRead = 0;
- this.totalZeroCopyBytesRead = 0;
- }
-
- private long totalBytesRead;
-
- private long totalLocalBytesRead;
-
- private long totalShortCircuitBytesRead;
-
- private long totalZeroCopyBytesRead;
- }
-
/**
* This variable tracks the number of failures since the start of the
* most recent user-facing operation. That is to say, it should be reset
@@ -767,116 +680,11 @@ public class DFSInputStream extends FSInputStream
return ( ret <= 0 ) ? -1 : (oneByteBuf[0] & 0xff);
}
- /**
- * Wraps different possible read implementations so that readBuffer can be
- * strategy-agnostic.
- */
- interface ReaderStrategy {
- int doRead(BlockReader blockReader, int off, int len)
- throws IOException;
-
- /**
- * Copy data from the src ByteBuffer into the read buffer.
- * @param src The src buffer where the data is copied from
- * @param offset Useful only when the ReadStrategy is based on a byte array.
- * Indicate the offset of the byte array for copy.
- * @param length Useful only when the ReadStrategy is based on a byte array.
- * Indicate the length of the data to copy.
- */
- int copyFrom(ByteBuffer src, int offset, int length);
- }
-
- protected void updateReadStatistics(ReadStatistics readStatistics,
- int nRead, BlockReader blockReader) {
- if (nRead <= 0) return;
- synchronized(infoLock) {
- if (blockReader.isShortCircuit()) {
- readStatistics.addShortCircuitBytes(nRead);
- } else if (blockReader.getNetworkDistance() == 0) {
- readStatistics.addLocalBytes(nRead);
- } else {
- readStatistics.addRemoteBytes(nRead);
- }
- }
- }
-
- /**
- * Used to read bytes into a byte[]
- */
- private class ByteArrayStrategy implements ReaderStrategy {
- final byte[] buf;
-
- public ByteArrayStrategy(byte[] buf) {
- this.buf = buf;
- }
-
- @Override
- public int doRead(BlockReader blockReader, int off, int len)
- throws IOException {
- int nRead = blockReader.read(buf, off, len);
- updateReadStatistics(readStatistics, nRead, blockReader);
- dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
- nRead);
- return nRead;
- }
-
- @Override
- public int copyFrom(ByteBuffer src, int offset, int length) {
- ByteBuffer writeSlice = src.duplicate();
- writeSlice.get(buf, offset, length);
- return length;
- }
- }
-
- /**
- * Used to read bytes into a user-supplied ByteBuffer
- */
- protected class ByteBufferStrategy implements ReaderStrategy {
- final ByteBuffer buf;
- ByteBufferStrategy(ByteBuffer buf) {
- this.buf = buf;
- }
-
- @Override
- public int doRead(BlockReader blockReader, int off, int len)
- throws IOException {
- int oldpos = buf.position();
- int oldlimit = buf.limit();
- boolean success = false;
- try {
- int ret = blockReader.read(buf);
- success = true;
- updateReadStatistics(readStatistics, ret, blockReader);
- dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
- ret);
- if (ret == 0) {
- DFSClient.LOG.warn("zero");
- }
- return ret;
- } finally {
- if (!success) {
- // Reset to original state so that retries work correctly.
- buf.position(oldpos);
- buf.limit(oldlimit);
- }
- }
- }
-
- @Override
- public int copyFrom(ByteBuffer src, int offset, int length) {
- ByteBuffer writeSlice = src.duplicate();
- int remaining = Math.min(buf.remaining(), writeSlice.remaining());
- writeSlice.limit(writeSlice.position() + remaining);
- buf.put(writeSlice);
- return remaining;
- }
- }
-
/* This is a used by regular read() and handles ChecksumExceptions.
* name readBuffer() is chosen to imply similarity to readBuffer() in
* ChecksumFileSystem
*/
- private synchronized int readBuffer(ReaderStrategy reader, int off, int len,
+ private synchronized int readBuffer(ReaderStrategy reader, int len,
CorruptedBlocks corruptedBlocks)
throws IOException {
IOException ioe;
@@ -892,7 +700,7 @@ public class DFSInputStream extends FSInputStream
while (true) {
// retry as many times as seekToNewSource allows.
try {
- return reader.doRead(blockReader, off, len);
+ return reader.readFromBlock(blockReader, len);
} catch ( ChecksumException ce ) {
DFSClient.LOG.warn("Found Checksum error for "
+ getCurrentBlock() + " from " + currentNode
@@ -927,13 +735,14 @@ public class DFSInputStream extends FSInputStream
}
}
- protected synchronized int readWithStrategy(ReaderStrategy strategy, int off,
- int len) throws IOException {
+ protected synchronized int readWithStrategy(ReaderStrategy strategy)
+ throws IOException {
dfsClient.checkOpen();
if (closed.get()) {
throw new IOException("Stream closed");
}
+ int len = strategy.getTargetLength();
CorruptedBlocks corruptedBlocks = new CorruptedBlocks();
failures = 0;
if (pos < getFileLength()) {
@@ -952,7 +761,7 @@ public class DFSInputStream extends FSInputStream
locatedBlocks.getFileLength() - pos);
}
}
- int result = readBuffer(strategy, off, realLen, corruptedBlocks);
+ int result = readBuffer(strategy, realLen, corruptedBlocks);
if (result >= 0) {
pos += result;
@@ -994,11 +803,12 @@ public class DFSInputStream extends FSInputStream
if (len == 0) {
return 0;
}
- ReaderStrategy byteArrayReader = new ByteArrayStrategy(buf);
+ ReaderStrategy byteArrayReader =
+ new ByteArrayStrategy(buf, off, len, readStatistics, dfsClient);
try (TraceScope scope =
dfsClient.newReaderTraceScope("DFSInputStream#byteArrayRead",
src, getPos(), len)) {
- int retLen = readWithStrategy(byteArrayReader, off, len);
+ int retLen = readWithStrategy(byteArrayReader);
if (retLen < len) {
dfsClient.addRetLenToReaderScope(scope, retLen);
}
@@ -1008,12 +818,13 @@ public class DFSInputStream extends FSInputStream
@Override
public synchronized int read(final ByteBuffer buf) throws IOException {
- ReaderStrategy byteBufferReader = new ByteBufferStrategy(buf);
+ ReaderStrategy byteBufferReader =
+ new ByteBufferStrategy(buf, readStatistics, dfsClient);
int reqLen = buf.remaining();
try (TraceScope scope =
dfsClient.newReaderTraceScope("DFSInputStream#byteBufferRead",
src, getPos(), reqLen)){
- int retLen = readWithStrategy(byteBufferReader, 0, reqLen);
+ int retLen = readWithStrategy(byteBufferReader);
if (retLen < reqLen) {
dfsClient.addRetLenToReaderScope(scope, retLen);
}
@@ -1221,7 +1032,7 @@ public class DFSInputStream extends FSInputStream
reader = getBlockReader(block, startInBlk, len, datanode.addr,
datanode.storageType, datanode.info);
int nread = reader.readAll(buf, offset, len);
- updateReadStatistics(readStatistics, nread, reader);
+ IOUtilsClient.updateReadStatistics(readStatistics, nread, reader);
dfsClient.updateFileSystemReadStats(
reader.getNetworkDistance(), nread);
if (nread != len) {
@@ -1721,18 +1532,14 @@ public class DFSInputStream extends FSInputStream
* Get statistics about the reads which this DFSInputStream has done.
*/
public ReadStatistics getReadStatistics() {
- synchronized(infoLock) {
- return new ReadStatistics(readStatistics);
- }
+ return new ReadStatistics(readStatistics);
}
/**
* Clear statistics about the reads which this DFSInputStream has done.
*/
public void clearReadStatistics() {
- synchronized(infoLock) {
- readStatistics.clear();
- }
+ readStatistics.clear();
}
public FileEncryptionInfo getFileEncryptionInfo() {
@@ -1759,7 +1566,8 @@ public class DFSInputStream extends FSInputStream
throws IOException {
synchronized (infoLock) {
this.cachingStrategy =
- new CachingStrategy.Builder(this.cachingStrategy).setReadahead(readahead).build();
+ new CachingStrategy.Builder(this.cachingStrategy).
+ setReadahead(readahead).build();
}
closeCurrentBlockReaders();
}
@@ -1769,7 +1577,8 @@ public class DFSInputStream extends FSInputStream
throws IOException {
synchronized (infoLock) {
this.cachingStrategy =
- new CachingStrategy.Builder(this.cachingStrategy).setDropBehind(dropBehind).build();
+ new CachingStrategy.Builder(this.cachingStrategy).
+ setDropBehind(dropBehind).build();
}
closeCurrentBlockReaders();
}
@@ -1883,9 +1692,7 @@ public class DFSInputStream extends FSInputStream
buffer.position((int)blockPos);
buffer.limit((int)(blockPos + length));
getExtendedReadBuffers().put(buffer, clientMmap);
- synchronized (infoLock) {
- readStatistics.addZeroCopyBytes(length);
- }
+ readStatistics.addZeroCopyBytes(length);
DFSClient.LOG.debug("readZeroCopy read {} bytes from offset {} via the "
+ "zero-copy read path. blockEnd = {}", length, curPos, blockEnd);
success = true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/793447f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index d93863c..9ca8005 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -359,11 +359,11 @@ public class DFSStripedInputStream extends DFSInputStream {
ExtendedBlock currentBlock,
CorruptedBlocks corruptedBlocks)
throws IOException {
- final int targetLength = strategy.buf.remaining();
+ final int targetLength = strategy.getTargetLength();
int length = 0;
try {
while (length < targetLength) {
- int ret = strategy.doRead(blockReader, 0, 0);
+ int ret = strategy.readFromBlock(blockReader);
if (ret < 0) {
throw new IOException("Unexpected EOS from the reader");
}
@@ -425,13 +425,14 @@ public class DFSStripedInputStream extends DFSInputStream {
}
@Override
- protected synchronized int readWithStrategy(ReaderStrategy strategy,
- int off, int len) throws IOException {
+ protected synchronized int readWithStrategy(ReaderStrategy strategy)
+ throws IOException {
dfsClient.checkOpen();
if (closed.get()) {
throw new IOException("Stream closed");
}
+ int len = strategy.getTargetLength();
CorruptedBlocks corruptedBlocks = new CorruptedBlocks();
if (pos < getFileLength()) {
try {
@@ -452,7 +453,7 @@ public class DFSStripedInputStream extends DFSInputStream {
if (!curStripeRange.include(getOffsetInBlockGroup())) {
readOneStripe(corruptedBlocks);
}
- int ret = copyToTargetBuf(strategy, off + result, realLen - result);
+ int ret = copyToTargetBuf(strategy, realLen - result);
result += ret;
pos += ret;
}
@@ -470,16 +471,14 @@ public class DFSStripedInputStream extends DFSInputStream {
/**
* Copy the data from {@link #curStripeBuf} into the given buffer
* @param strategy the ReaderStrategy containing the given buffer
- * @param offset the offset of the given buffer. Used only when strategy is
- * a ByteArrayStrategy
* @param length target length
* @return number of bytes copied
*/
- private int copyToTargetBuf(ReaderStrategy strategy, int offset, int length) {
+ private int copyToTargetBuf(ReaderStrategy strategy, int length) {
final long offsetInBlk = getOffsetInBlockGroup();
int bufOffset = getStripedBufOffset(offsetInBlk);
curStripeBuf.position(bufOffset);
- return strategy.copyFrom(curStripeBuf, offset,
+ return strategy.readFromBuffer(curStripeBuf,
Math.min(length, curStripeBuf.remaining()));
}
@@ -700,7 +699,8 @@ public class DFSStripedInputStream extends DFSInputStream {
private ByteBufferStrategy[] getReadStrategies(StripingChunk chunk) {
if (chunk.byteBuffer != null) {
- ByteBufferStrategy strategy = new ByteBufferStrategy(chunk.byteBuffer);
+ ByteBufferStrategy strategy =
+ new ByteBufferStrategy(chunk.byteBuffer, readStatistics, dfsClient);
return new ByteBufferStrategy[]{strategy};
} else {
ByteBufferStrategy[] strategies =
@@ -708,7 +708,8 @@ public class DFSStripedInputStream extends DFSInputStream {
for (int i = 0; i < strategies.length; i++) {
ByteBuffer buffer = ByteBuffer.wrap(chunk.byteArray.buf(),
chunk.byteArray.getOffsets()[i], chunk.byteArray.getLengths()[i]);
- strategies[i] = new ByteBufferStrategy(buffer);
+ strategies[i] =
+ new ByteBufferStrategy(buffer, readStatistics, dfsClient);
}
return strategies;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/793447f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReadStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReadStatistics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReadStatistics.java
new file mode 100644
index 0000000..59b1418
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReadStatistics.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+/**
+ * A utility class that maintains statistics for reading.
+ */
+public class ReadStatistics {
+ private long totalBytesRead;
+ private long totalLocalBytesRead;
+ private long totalShortCircuitBytesRead;
+ private long totalZeroCopyBytesRead;
+
+ public ReadStatistics() {
+ clear();
+ }
+
+ public ReadStatistics(ReadStatistics rhs) {
+ this.totalBytesRead = rhs.getTotalBytesRead();
+ this.totalLocalBytesRead = rhs.getTotalLocalBytesRead();
+ this.totalShortCircuitBytesRead = rhs.getTotalShortCircuitBytesRead();
+ this.totalZeroCopyBytesRead = rhs.getTotalZeroCopyBytesRead();
+ }
+
+ /**
+ * @return The total bytes read. This will always be at least as
+ * high as the other numbers, since it includes all of them.
+ */
+ public synchronized long getTotalBytesRead() {
+ return totalBytesRead;
+ }
+
+ /**
+ * @return The total local bytes read. This will always be at least
+ * as high as totalShortCircuitBytesRead, since all short-circuit
+ * reads are also local.
+ */
+ public synchronized long getTotalLocalBytesRead() {
+ return totalLocalBytesRead;
+ }
+
+ /**
+ * @return The total short-circuit local bytes read.
+ */
+ public synchronized long getTotalShortCircuitBytesRead() {
+ return totalShortCircuitBytesRead;
+ }
+
+ /**
+ * @return The total number of zero-copy bytes read.
+ */
+ public synchronized long getTotalZeroCopyBytesRead() {
+ return totalZeroCopyBytesRead;
+ }
+
+ /**
+ * @return The total number of bytes read which were not local.
+ */
+ public synchronized long getRemoteBytesRead() {
+ return totalBytesRead - totalLocalBytesRead;
+ }
+
+ public synchronized void addRemoteBytes(long amt) {
+ this.totalBytesRead += amt;
+ }
+
+ public synchronized void addLocalBytes(long amt) {
+ this.totalBytesRead += amt;
+ this.totalLocalBytesRead += amt;
+ }
+
+ public synchronized void addShortCircuitBytes(long amt) {
+ this.totalBytesRead += amt;
+ this.totalLocalBytesRead += amt;
+ this.totalShortCircuitBytesRead += amt;
+ }
+
+ public synchronized void addZeroCopyBytes(long amt) {
+ this.totalBytesRead += amt;
+ this.totalLocalBytesRead += amt;
+ this.totalShortCircuitBytesRead += amt;
+ this.totalZeroCopyBytesRead += amt;
+ }
+
+ public synchronized void clear() {
+ this.totalBytesRead = 0;
+ this.totalLocalBytesRead = 0;
+ this.totalShortCircuitBytesRead = 0;
+ this.totalZeroCopyBytesRead = 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/793447f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java
new file mode 100644
index 0000000..d75a8ef
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics;
+
+/**
+ * Wraps different possible read implementations so that callers can be
+ * strategy-agnostic.
+ */
+interface ReaderStrategy {
+ /**
+ * Read from a block using the blockReader.
+ * @param blockReader
+ * @return number of bytes read
+ * @throws IOException
+ */
+ int readFromBlock(BlockReader blockReader) throws IOException;
+
+ /**
+ * Read from a block using the blockReader with desired length to read.
+ * @param blockReader
+ * @param length number of bytes desired to read, not ensured
+ * @return number of bytes read
+ * @throws IOException
+ */
+ int readFromBlock(BlockReader blockReader, int length) throws IOException;
+
+ /**
+ * Read or copy from a src buffer.
+ * @param src
+ * @return number of bytes copied
+ * Note: the position of the src buffer is not changed after the call
+ */
+ int readFromBuffer(ByteBuffer src);
+
+ /**
+ * Read or copy length of data bytes from a src buffer with desired length.
+ * @param src
+ * @return number of bytes copied
+ * Note: the position of the src buffer is not changed after the call
+ */
+ int readFromBuffer(ByteBuffer src, int length);
+
+ /**
+ * @return the target read buffer that reads data into.
+ */
+ ByteBuffer getReadBuffer();
+
+ /**
+ * @return the target length to read.
+ */
+ int getTargetLength();
+}
+
+/**
+ * Used to read bytes into a byte array buffer. Note it's not thread-safe
+ * and the behavior is not defined if concurrently operated.
+ */
+class ByteArrayStrategy implements ReaderStrategy {
+ private final DFSClient dfsClient;
+ private final ReadStatistics readStatistics;
+ private final byte[] readBuf;
+ private int offset;
+ private final int targetLength;
+
+ /**
+ * The constructor.
+ * @param readBuf target buffer to read into
+ * @param offset offset into the buffer
+ * @param targetLength target length of data
+ * @param readStatistics statistics counter
+ */
+ public ByteArrayStrategy(byte[] readBuf, int offset, int targetLength,
+ ReadStatistics readStatistics,
+ DFSClient dfsClient) {
+ this.readBuf = readBuf;
+ this.offset = offset;
+ this.targetLength = targetLength;
+ this.readStatistics = readStatistics;
+ this.dfsClient = dfsClient;
+ }
+
+ @Override
+ public ByteBuffer getReadBuffer() {
+ return ByteBuffer.wrap(readBuf, offset, targetLength);
+ }
+
+ @Override
+ public int getTargetLength() {
+ return targetLength;
+ }
+
+ @Override
+ public int readFromBlock(BlockReader blockReader) throws IOException {
+ return readFromBlock(blockReader, targetLength);
+ }
+
+ @Override
+ public int readFromBlock(BlockReader blockReader,
+ int length) throws IOException {
+ int nRead = blockReader.read(readBuf, offset, length);
+ if (nRead > 0) {
+ updateReadStatistics(readStatistics, nRead, blockReader);
+ dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
+ nRead);
+ offset += nRead;
+ }
+ return nRead;
+ }
+
+ @Override
+ public int readFromBuffer(ByteBuffer src) {
+ return readFromBuffer(src, src.remaining());
+ }
+
+ @Override
+ public int readFromBuffer(ByteBuffer src, int length) {
+ ByteBuffer dup = src.duplicate();
+ dup.get(readBuf, offset, length);
+ offset += length;
+ return length;
+ }
+}
+
+/**
+ * Used to read bytes into a user-supplied ByteBuffer. Note it's not thread-safe
+ * and the behavior is not defined if concurrently operated. When read operation
+ * is performed, the position of the underlying byte buffer will move forward as
+ * stated in ByteBufferReadable#read(ByteBuffer buf) method.
+ */
+class ByteBufferStrategy implements ReaderStrategy {
+ private final DFSClient dfsClient;
+ private final ReadStatistics readStatistics;
+ private final ByteBuffer readBuf;
+ private final int targetLength;
+
+ /**
+ * The constructor.
+ * @param readBuf target buffer to read into
+ * @param readStatistics statistics counter
+ */
+ ByteBufferStrategy(ByteBuffer readBuf,
+ ReadStatistics readStatistics,
+ DFSClient dfsClient) {
+ this.readBuf = readBuf;
+ this.targetLength = readBuf.remaining();
+ this.readStatistics = readStatistics;
+ this.dfsClient = dfsClient;
+ }
+
+ @Override
+ public ByteBuffer getReadBuffer() {
+ return readBuf;
+ }
+
+ @Override
+ public int readFromBlock(BlockReader blockReader) throws IOException {
+ return readFromBlock(blockReader, readBuf.remaining());
+ }
+
+ @Override
+ public int readFromBlock(BlockReader blockReader,
+ int length) throws IOException {
+ ByteBuffer tmpBuf = readBuf.duplicate();
+ tmpBuf.limit(tmpBuf.position() + length);
+ int nRead = blockReader.read(readBuf.slice());
+ // Only when data are read, update the position
+ if (nRead > 0) {
+ readBuf.position(readBuf.position() + nRead);
+ updateReadStatistics(readStatistics, nRead, blockReader);
+ dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
+ nRead);
+ }
+
+ return nRead;
+ }
+
+ @Override
+ public int getTargetLength() {
+ return targetLength;
+ }
+
+ @Override
+ public int readFromBuffer(ByteBuffer src) {
+ return readFromBuffer(src, src.remaining());
+ }
+
+ @Override
+ public int readFromBuffer(ByteBuffer src, int length) {
+ ByteBuffer dup = src.duplicate();
+ int newLen = Math.min(readBuf.remaining(), dup.remaining());
+ newLen = Math.min(newLen, length);
+ dup.limit(dup.position() + newLen);
+ readBuf.put(dup);
+ return newLen;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/793447f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
index 8e95451..02e5deb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.crypto.CryptoInputStream;
import org.apache.hadoop.hdfs.DFSInputStream;
+import org.apache.hadoop.hdfs.ReadStatistics;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -103,7 +104,7 @@ public class HdfsDataInputStream extends FSDataInputStream {
* be higher than you would expect just by adding up the number of
* bytes read through HdfsDataInputStream.
*/
- public DFSInputStream.ReadStatistics getReadStatistics() {
+ public ReadStatistics getReadStatistics() {
return getDFSInputStream().getReadStatistics();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/793447f7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/IOUtilsClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/IOUtilsClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/IOUtilsClient.java
index 56f8ecc..71596f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/IOUtilsClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/IOUtilsClient.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.util;
+import org.apache.hadoop.hdfs.BlockReader;
+import org.apache.hadoop.hdfs.ReadStatistics;
import org.slf4j.Logger;
import java.io.IOException;
@@ -43,4 +45,18 @@ public class IOUtilsClient {
}
}
+ public static void updateReadStatistics(ReadStatistics readStatistics,
+ int nRead, BlockReader blockReader) {
+ if (nRead <= 0) {
+ return;
+ }
+
+ if (blockReader.isShortCircuit()) {
+ readStatistics.addShortCircuitBytes(nRead);
+ } else if (blockReader.getNetworkDistance() == 0) {
+ readStatistics.addLocalBytes(nRead);
+ } else {
+ readStatistics.addRemoteBytes(nRead);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/793447f7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestExternalBlockReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestExternalBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestExternalBlockReader.java
index 5c2b6da..8acf4bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestExternalBlockReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestExternalBlockReader.java
@@ -23,7 +23,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -40,7 +39,8 @@ import java.util.LinkedList;
import java.util.UUID;
public class TestExternalBlockReader {
- private static final Log LOG = LogFactory.getLog(TestExternalBlockReader.class);
+ private static final Log LOG =
+ LogFactory.getLog(TestExternalBlockReader.class);
private static long SEED = 1234;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/52] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by aw...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
deleted file mode 100644
index 5ba1871..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.IOException;
-import java.net.URI;
-import java.nio.file.AccessDeniedException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.junit.rules.Timeout;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSCredentialsProviderChain;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
-import static org.junit.Assert.*;
-
-/**
- * Tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic.
- *
- */
-public class TestS3AAWSCredentialsProvider {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class);
-
- @Rule
- public Timeout testTimeout = new Timeout(1 * 60 * 1000);
-
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- /**
- * Declare what exception to raise, and the text which must be found
- * in it.
- * @param exceptionClass class of exception
- * @param text text in exception
- */
- private void expectException(Class<? extends Throwable> exceptionClass,
- String text) {
- exception.expect(exceptionClass);
- exception.expectMessage(text);
- }
-
- @Test
- public void testBadConfiguration() throws IOException {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER, "no.such.class");
- try {
- createFailingFS(conf);
- } catch (IOException e) {
- if (!(e.getCause() instanceof ClassNotFoundException)) {
- LOG.error("Unexpected nested cause: {} in {}", e.getCause(), e, e);
- throw e;
- }
- }
- }
-
- /**
- * Create a filesystem, expect it to fail by raising an IOException.
- * Raises an assertion exception if in fact the FS does get instantiated.
- * @param conf configuration
- * @throws IOException an expected exception.
- */
- private void createFailingFS(Configuration conf) throws IOException {
- S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf);
- fs.listStatus(new Path("/"));
- fail("Expected exception - got " + fs);
- }
-
- static class BadCredentialsProvider implements AWSCredentialsProvider {
-
- @SuppressWarnings("unused")
- public BadCredentialsProvider(URI name, Configuration conf) {
- }
-
- @Override
- public AWSCredentials getCredentials() {
- return new BasicAWSCredentials("bad_key", "bad_secret");
- }
-
- @Override
- public void refresh() {
- }
- }
-
- @Test
- public void testBadCredentials() throws Exception {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER, BadCredentialsProvider.class.getName());
- try {
- createFailingFS(conf);
- } catch (AccessDeniedException e) {
- // expected
- }
- }
-
- static class GoodCredentialsProvider extends AWSCredentialsProviderChain {
-
- @SuppressWarnings("unused")
- public GoodCredentialsProvider(URI name, Configuration conf) {
- super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
- conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider());
- }
- }
-
- @Test
- public void testGoodProvider() throws Exception {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName());
- S3ATestUtils.createTestFileSystem(conf);
- }
-
- @Test
- public void testAnonymousProvider() throws Exception {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER,
- AnonymousAWSCredentialsProvider.class.getName());
- Path testFile = new Path(
- conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
- S3ATestUtils.useCSVDataEndpoint(conf);
- FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
- assertNotNull(fs);
- assertTrue(fs instanceof S3AFileSystem);
- FileStatus stat = fs.getFileStatus(testFile);
- assertNotNull(stat);
- assertEquals(testFile, stat.getPath());
- }
-
- /**
- * A credential provider whose constructor signature doesn't match.
- */
- static class ConstructorSignatureErrorProvider
- implements AWSCredentialsProvider {
-
- @SuppressWarnings("unused")
- public ConstructorSignatureErrorProvider(String str) {
- }
-
- @Override
- public AWSCredentials getCredentials() {
- return null;
- }
-
- @Override
- public void refresh() {
- }
- }
-
- /**
- * A credential provider whose constructor raises an NPE.
- */
- static class ConstructorFailureProvider
- implements AWSCredentialsProvider {
-
- @SuppressWarnings("unused")
- public ConstructorFailureProvider() {
- throw new NullPointerException("oops");
- }
-
- @Override
- public AWSCredentials getCredentials() {
- return null;
- }
-
- @Override
- public void refresh() {
- }
- }
-
- @Test
- public void testProviderWrongClass() throws Exception {
- expectProviderInstantiationFailure(this.getClass().getName(),
- NOT_AWS_PROVIDER);
- }
-
- @Test
- public void testProviderNotAClass() throws Exception {
- expectProviderInstantiationFailure("NoSuchClass",
- "ClassNotFoundException");
- }
-
- private void expectProviderInstantiationFailure(String option,
- String expectedErrorText) throws IOException {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER, option);
- Path testFile = new Path(
- conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
- expectException(IOException.class, expectedErrorText);
- URI uri = testFile.toUri();
- S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
- }
-
- @Test
- public void testProviderConstructorError() throws Exception {
- expectProviderInstantiationFailure(
- ConstructorSignatureErrorProvider.class.getName(),
- CONSTRUCTOR_EXCEPTION);
- }
-
- @Test
- public void testProviderFailureError() throws Exception {
- expectProviderInstantiationFailure(
- ConstructorFailureProvider.class.getName(),
- INSTANTIATION_EXCEPTION);
- }
-
- @Test
- public void testInstantiationChain() throws Throwable {
- Configuration conf = new Configuration();
- conf.set(AWS_CREDENTIALS_PROVIDER,
- TemporaryAWSCredentialsProvider.NAME
- + ", \t" + SimpleAWSCredentialsProvider.NAME
- + " ,\n " + AnonymousAWSCredentialsProvider.NAME);
- Path testFile = new Path(
- conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
-
- URI uri = testFile.toUri();
- S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockingThreadPool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockingThreadPool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockingThreadPool.java
deleted file mode 100644
index bd738b2..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockingThreadPool.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-/**
- * Demonstrate that the threadpool blocks additional client requests if
- * its queue is full (rather than throwing an exception) by initiating an
- * upload consisting of 4 parts with 2 threads and 1 spot in the queue. The
- * 4th part should not trigger an exception as it would with a
- * non-blocking threadpool.
- */
-public class TestS3ABlockingThreadPool {
-
- private Configuration conf;
- private S3AFileSystem fs;
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- protected Path getTestPath() {
- return new Path("/tests3a");
- }
-
- @Before
- public void setUp() throws Exception {
- conf = new Configuration();
- conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024);
- conf.setLong(Constants.MULTIPART_SIZE, 5 * 1024 * 1024);
- conf.setInt(Constants.MAX_THREADS, 2);
- conf.setInt(Constants.MAX_TOTAL_TASKS, 1);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(getTestPath(), true);
- }
- }
-
- @Test
- public void testRegularMultiPartUpload() throws Exception {
- fs = S3ATestUtils.createTestFileSystem(conf);
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 *
- 1024);
- }
-
- @Test
- public void testFastMultiPartUpload() throws Exception {
- conf.setBoolean(Constants.FAST_UPLOAD, true);
- fs = S3ATestUtils.createTestFileSystem(conf);
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 *
- 1024);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlocksize.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlocksize.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlocksize.java
deleted file mode 100644
index 76fbf99..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlocksize.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.fileStatsToString;
-
-public class TestS3ABlocksize extends AbstractFSContractTestBase {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3ABlocksize.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Test
- @SuppressWarnings("deprecation")
- public void testBlockSize() throws Exception {
- FileSystem fs = getFileSystem();
- long defaultBlockSize = fs.getDefaultBlockSize();
- assertEquals("incorrect blocksize",
- S3AFileSystem.DEFAULT_BLOCKSIZE, defaultBlockSize);
- long newBlockSize = defaultBlockSize * 2;
- fs.getConf().setLong(Constants.FS_S3A_BLOCK_SIZE, newBlockSize);
-
- Path dir = path("testBlockSize");
- Path file = new Path(dir, "file");
- createFile(fs, file, true, dataset(1024, 'a', 'z' - 'a'));
- FileStatus fileStatus = fs.getFileStatus(file);
- assertEquals("Double default block size in stat(): " + fileStatus,
- newBlockSize,
- fileStatus.getBlockSize());
-
- // check the listing & assert that the block size is picked up by
- // this route too.
- boolean found = false;
- FileStatus[] listing = fs.listStatus(dir);
- for (FileStatus stat : listing) {
- LOG.info("entry: {}", stat);
- if (file.equals(stat.getPath())) {
- found = true;
- assertEquals("Double default block size in ls(): " + stat,
- newBlockSize,
- stat.getBlockSize());
- }
- }
- assertTrue("Did not find " + fileStatsToString(listing, ", "), found);
- }
-
- @Test
- public void testRootFileStatusHasBlocksize() throws Throwable {
- FileSystem fs = getFileSystem();
- FileStatus status = fs.getFileStatus(new Path("/"));
- assertTrue("Invalid root blocksize",
- status.getBlockSize() >= 0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
deleted file mode 100644
index 29bfd59..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AConfiguration.java
+++ /dev/null
@@ -1,429 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.reflect.FieldUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3native.S3xLoginHelper;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.net.URI;
-
-import org.apache.hadoop.security.ProviderUtils;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.http.HttpStatus;
-import org.junit.rules.TemporaryFolder;
-
-public class TestS3AConfiguration {
- private static final String EXAMPLE_ID = "AKASOMEACCESSKEY";
- private static final String EXAMPLE_KEY =
- "RGV0cm9pdCBSZ/WQgY2xl/YW5lZCB1cAEXAMPLE";
-
- private Configuration conf;
- private S3AFileSystem fs;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AConfiguration.class);
-
- private static final String TEST_ENDPOINT = "test.fs.s3a.endpoint";
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Rule
- public final TemporaryFolder tempDir = new TemporaryFolder();
-
- /**
- * Test if custom endpoint is picked up.
- * <p/>
- * The test expects TEST_ENDPOINT to be defined in the Configuration
- * describing the endpoint of the bucket to which TEST_FS_S3A_NAME points
- * (f.i. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland).
- * Evidently, the bucket has to be hosted in the region denoted by the
- * endpoint for the test to succeed.
- * <p/>
- * More info and the list of endpoint identifiers:
- * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
- *
- * @throws Exception
- */
- @Test
- public void testEndpoint() throws Exception {
- conf = new Configuration();
- String endpoint = conf.getTrimmed(TEST_ENDPOINT, "");
- if (endpoint.isEmpty()) {
- LOG.warn("Custom endpoint test skipped as " + TEST_ENDPOINT + "config " +
- "setting was not detected");
- } else {
- conf.set(Constants.ENDPOINT, endpoint);
- fs = S3ATestUtils.createTestFileSystem(conf);
- AmazonS3Client s3 = fs.getAmazonS3Client();
- String endPointRegion = "";
- // Differentiate handling of "s3-" and "s3." based endpoint identifiers
- String[] endpointParts = StringUtils.split(endpoint, '.');
- if (endpointParts.length == 3) {
- endPointRegion = endpointParts[0].substring(3);
- } else if (endpointParts.length == 4) {
- endPointRegion = endpointParts[1];
- } else {
- fail("Unexpected endpoint");
- }
- assertEquals("Endpoint config setting and bucket location differ: ",
- endPointRegion, s3.getBucketLocation(fs.getUri().getHost()));
- }
- }
-
- @Test
- public void testProxyConnection() throws Exception {
- conf = new Configuration();
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.set(Constants.PROXY_HOST, "127.0.0.1");
- conf.setInt(Constants.PROXY_PORT, 1);
- String proxy =
- conf.get(Constants.PROXY_HOST) + ":" + conf.get(Constants.PROXY_PORT);
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server at " + proxy);
- } catch (AWSClientIOException e) {
- // expected
- }
- }
-
- @Test
- public void testProxyPortWithoutHost() throws Exception {
- conf = new Configuration();
- conf.unset(Constants.PROXY_HOST);
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.setInt(Constants.PROXY_PORT, 1);
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a proxy configuration error");
- } catch (IllegalArgumentException e) {
- String msg = e.toString();
- if (!msg.contains(Constants.PROXY_HOST) &&
- !msg.contains(Constants.PROXY_PORT)) {
- throw e;
- }
- }
- }
-
- @Test
- public void testAutomaticProxyPortSelection() throws Exception {
- conf = new Configuration();
- conf.unset(Constants.PROXY_PORT);
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.set(Constants.PROXY_HOST, "127.0.0.1");
- conf.set(Constants.SECURE_CONNECTIONS, "true");
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server");
- } catch (AWSClientIOException e) {
- // expected
- }
- conf.set(Constants.SECURE_CONNECTIONS, "false");
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server");
- } catch (AWSClientIOException e) {
- // expected
- }
- }
-
- @Test
- public void testUsernameInconsistentWithPassword() throws Exception {
- conf = new Configuration();
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.set(Constants.PROXY_HOST, "127.0.0.1");
- conf.setInt(Constants.PROXY_PORT, 1);
- conf.set(Constants.PROXY_USERNAME, "user");
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server");
- } catch (IllegalArgumentException e) {
- String msg = e.toString();
- if (!msg.contains(Constants.PROXY_USERNAME) &&
- !msg.contains(Constants.PROXY_PASSWORD)) {
- throw e;
- }
- }
- conf = new Configuration();
- conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
- conf.set(Constants.PROXY_HOST, "127.0.0.1");
- conf.setInt(Constants.PROXY_PORT, 1);
- conf.set(Constants.PROXY_PASSWORD, "password");
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fail("Expected a connection error for proxy server");
- } catch (IllegalArgumentException e) {
- String msg = e.toString();
- if (!msg.contains(Constants.PROXY_USERNAME) &&
- !msg.contains(Constants.PROXY_PASSWORD)) {
- throw e;
- }
- }
- }
-
- @Test
- public void testCredsFromCredentialProvider() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- provisionAccessKeys(conf);
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
- }
-
- void provisionAccessKeys(final Configuration conf) throws Exception {
- // add our creds to the provider
- final CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
- provider.createCredentialEntry(Constants.ACCESS_KEY,
- EXAMPLE_ID.toCharArray());
- provider.createCredentialEntry(Constants.SECRET_KEY,
- EXAMPLE_KEY.toCharArray());
- provider.flush();
- }
-
- @Test
- public void testCredsFromUserInfo() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- provisionAccessKeys(conf);
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
- URI uriWithUserInfo = new URI("s3a://123:456@foobar");
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
- assertEquals("AccessKey incorrect.", "123", creds.getUser());
- assertEquals("SecretKey incorrect.", "456", creds.getPassword());
- }
-
- @Test
- public void testIDFromUserInfoSecretFromCredentialProvider()
- throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- provisionAccessKeys(conf);
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
- URI uriWithUserInfo = new URI("s3a://123@foobar");
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
- assertEquals("AccessKey incorrect.", "123", creds.getUser());
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
- }
-
- @Test
- public void testSecretFromCredentialProviderIDFromConfig() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- // add our creds to the provider
- final CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
- provider.createCredentialEntry(Constants.SECRET_KEY,
- EXAMPLE_KEY.toCharArray());
- provider.flush();
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID);
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
- }
-
- @Test
- public void testIDFromCredentialProviderSecretFromConfig() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- jks.toString());
-
- // add our creds to the provider
- final CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
- provider.createCredentialEntry(Constants.ACCESS_KEY,
- EXAMPLE_ID.toCharArray());
- provider.flush();
-
- conf.set(Constants.SECRET_KEY, EXAMPLE_KEY);
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
- }
-
- @Test
- public void testExcludingS3ACredentialProvider() throws Exception {
- // set up conf to have a cred provider
- final Configuration conf = new Configuration();
- final File file = tempDir.newFile("test.jks");
- final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
- file.toURI());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
- "jceks://s3a/foobar," + jks.toString());
-
- // first make sure that the s3a based provider is removed
- Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
- conf, S3AFileSystem.class);
- String newPath = conf.get(
- CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
- assertFalse("Provider Path incorrect", newPath.contains("s3a://"));
-
- // now let's make sure the new path is created by the S3AFileSystem
- // and the integration still works. Let's provision the keys through
- // the altered configuration instance and then try and access them
- // using the original config with the s3a provider in the path.
- provisionAccessKeys(c);
-
- conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
- URI uriWithUserInfo = new URI("s3a://123:456@foobar");
- S3xLoginHelper.Login creds =
- S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
- assertEquals("AccessKey incorrect.", "123", creds.getUser());
- assertEquals("SecretKey incorrect.", "456", creds.getPassword());
-
- }
-
- @Test
- public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() throws Exception {
-
- conf = new Configuration();
- conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true));
- assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false));
-
- try {
- fs = S3ATestUtils.createTestFileSystem(conf);
- assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
- assertNotNull(s3);
- S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class,
- "clientOptions");
- assertTrue("Expected to find path style access to be switched on!",
- clientOptions.isPathStyleAccess());
- byte[] file = ContractTestUtils.toAsciiByteArray("test file");
- ContractTestUtils.writeAndRead(fs, new Path("/path/style/access/testFile"), file, file.length, conf.getInt(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true);
- } catch (final AWSS3IOException e) {
- LOG.error("Caught exception: ", e);
- // Catch/pass standard path style access behaviour when live bucket
- // isn't in the same region as the s3 client default. See
- // http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
- assertEquals(e.getStatusCode(), HttpStatus.SC_MOVED_PERMANENTLY);
- }
- }
-
- @Test
- public void testDefaultUserAgent() throws Exception {
- conf = new Configuration();
- fs = S3ATestUtils.createTestFileSystem(conf);
- assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
- assertNotNull(s3);
- ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
- "clientConfiguration");
- assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent());
- }
-
- @Test
- public void testCustomUserAgent() throws Exception {
- conf = new Configuration();
- conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
- fs = S3ATestUtils.createTestFileSystem(conf);
- assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
- assertNotNull(s3);
- ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
- "clientConfiguration");
- assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
- awsConf.getUserAgent());
- }
-
- /**
- * Reads and returns a field from an object using reflection. If the field
- * cannot be found, is null, or is not the expected type, then this method
- * fails the test.
- *
- * @param target object to read
- * @param fieldType type of field to read, which will also be the return type
- * @param fieldName name of field to read
- * @return field that was read
- * @throws IllegalAccessException if access not allowed
- */
- private static <T> T getField(Object target, Class<T> fieldType,
- String fieldName) throws IllegalAccessException {
- Object obj = FieldUtils.readField(target, fieldName, true);
- assertNotNull(String.format(
- "Could not read field named %s in object with class %s.", fieldName,
- target.getClass().getName()), obj);
- assertTrue(String.format(
- "Unexpected type found for field named %s, expected %s, actual %s.",
- fieldName, fieldType.getName(), obj.getClass().getName()),
- fieldType.isAssignableFrom(obj.getClass()));
- return fieldType.cast(obj);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ACredentialsInURL.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ACredentialsInURL.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ACredentialsInURL.java
deleted file mode 100644
index 8cb7c0f..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ACredentialsInURL.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URLEncoder;
-import java.nio.file.AccessDeniedException;
-
-import static org.apache.hadoop.fs.s3a.S3ATestConstants.TEST_FS_S3A_NAME;
-
-/**
- * Tests that credentials can go into the URL. This includes a valid
- * set, and a check that an invalid set do at least get stripped out
- * of the final URI
- */
-public class TestS3ACredentialsInURL extends Assert {
- private S3AFileSystem fs;
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3ACredentialsInURL.class);
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @After
- public void teardown() {
- IOUtils.closeStream(fs);
- }
-
- /**
- * Test instantiation.
- * @throws Throwable
- */
- @Test
- public void testInstantiateFromURL() throws Throwable {
-
- Configuration conf = new Configuration();
- String accessKey = conf.get(Constants.ACCESS_KEY);
- String secretKey = conf.get(Constants.SECRET_KEY);
- String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, "");
- Assume.assumeNotNull(fsname, accessKey, secretKey);
- URI original = new URI(fsname);
- URI secretsURI = createUriWithEmbeddedSecrets(original,
- accessKey, secretKey);
- if (secretKey.contains("/")) {
- assertTrue("test URI encodes the / symbol", secretsURI.toString().
- contains("%252F"));
- }
- if (secretKey.contains("+")) {
- assertTrue("test URI encodes the + symbol", secretsURI.toString().
- contains("%252B"));
- }
- assertFalse("Does not contain secrets", original.equals(secretsURI));
-
- conf.set(TEST_FS_S3A_NAME, secretsURI.toString());
- conf.unset(Constants.ACCESS_KEY);
- conf.unset(Constants.SECRET_KEY);
- fs = S3ATestUtils.createTestFileSystem(conf);
- String fsURI = fs.getUri().toString();
- assertFalse("FS URI contains a @ symbol", fsURI.contains("@"));
- assertFalse("FS URI contains a % symbol", fsURI.contains("%"));
- if (!original.toString().startsWith(fsURI)) {
- fail("Filesystem URI does not match original");
- }
- validate("original path", new Path(original));
- validate("bare path", new Path("/"));
- validate("secrets path", new Path(secretsURI));
- }
-
- private void validate(String text, Path path) throws IOException {
- try {
- fs.canonicalizeUri(path.toUri());
- fs.checkPath(path);
- assertTrue(text + " Not a directory",
- fs.getFileStatus(new Path("/")).isDirectory());
- fs.globStatus(path);
- } catch (AssertionError e) {
- throw e;
- } catch (Exception e) {
- LOG.debug("{} failure: {}", text, e, e);
- fail(text + " Test failed");
- }
- }
-
- /**
- * Set up some invalid credentials, verify login is rejected.
- * @throws Throwable
- */
- @Test
- public void testInvalidCredentialsFail() throws Throwable {
- Configuration conf = new Configuration();
- String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, "");
- Assume.assumeNotNull(fsname);
- URI original = new URI(fsname);
- URI testURI = createUriWithEmbeddedSecrets(original, "user", "//");
-
- conf.set(TEST_FS_S3A_NAME, testURI.toString());
- fs = S3ATestUtils.createTestFileSystem(conf);
- try {
- S3AFileStatus status = fs.getFileStatus(new Path("/"));
- fail("Expected an AccessDeniedException, got " + status);
- } catch (AccessDeniedException e) {
- // expected
- }
-
- }
-
- private URI createUriWithEmbeddedSecrets(URI original,
- String accessKey,
- String secretKey) throws UnsupportedEncodingException {
- String encodedSecretKey = URLEncoder.encode(secretKey, "UTF-8");
- String formattedString = String.format("%s://%s:%s@%s/%s/",
- original.getScheme(),
- accessKey,
- encodedSecretKey,
- original.getHost(),
- original.getPath());
- URI testURI;
- try {
- testURI = new Path(formattedString).toUri();
- } catch (IllegalArgumentException e) {
- // inner cause is stripped to keep any secrets out of stack traces
- throw new IllegalArgumentException("Could not encode Path");
- }
- return testURI;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryption.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryption.java
deleted file mode 100644
index 43a26ff..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryption.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-
-/**
- * Test whether or not encryption works by turning it on. Some checks
- * are made for different file sizes as there have been reports that the
- * file length may be rounded up to match word boundaries.
- */
-public class TestS3AEncryption extends AbstractS3ATestBase {
- private static final String AES256 = Constants.SERVER_SIDE_ENCRYPTION_AES256;
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- S3ATestUtils.disableFilesystemCaching(conf);
- conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
- AES256);
- return conf;
- }
-
- private static final int[] SIZES = {
- 0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 10 - 3, 2 ^ 11 - 2, 2 ^ 12 - 1
- };
-
- @Override
- public void teardown() throws Exception {
- super.teardown();
- IOUtils.closeStream(getFileSystem());
- }
-
- @Test
- public void testEncryption() throws Throwable {
- for (int size: SIZES) {
- validateEncryptionForFilesize(size);
- }
- }
-
- @Test
- public void testEncryptionOverRename() throws Throwable {
- skipIfEncryptionTestsDisabled(getConfiguration());
- Path src = path(createFilename(1024));
- byte[] data = dataset(1024, 'a', 'z');
- S3AFileSystem fs = getFileSystem();
- writeDataset(fs, src, data, data.length, 1024 * 1024, true);
- ContractTestUtils.verifyFileContents(fs, src, data);
- Path dest = path(src.getName() + "-copy");
- fs.rename(src, dest);
- ContractTestUtils.verifyFileContents(fs, dest, data);
- assertEncrypted(dest);
- }
-
- protected void validateEncryptionForFilesize(int len) throws IOException {
- skipIfEncryptionTestsDisabled(getConfiguration());
- describe("Create an encrypted file of size " + len);
- String src = createFilename(len);
- Path path = writeThenReadFile(src, len);
- assertEncrypted(path);
- rm(getFileSystem(), path, false, false);
- }
-
- private String createFilename(int len) {
- return String.format("%s-%04x", methodName.getMethodName(), len);
- }
-
- /**
- * Assert that at path references an encrypted blob.
- * @param path path
- * @throws IOException on a failure
- */
- private void assertEncrypted(Path path) throws IOException {
- ObjectMetadata md = getFileSystem().getObjectMetadata(path);
- assertEquals(AES256, md.getSSEAlgorithm());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionAlgorithmPropagation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionAlgorithmPropagation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionAlgorithmPropagation.java
deleted file mode 100644
index 04ca830..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionAlgorithmPropagation.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-
-/**
- * Test whether or not encryption settings propagate by choosing an invalid
- * one. We expect the write to fail with a 400 bad request error
- */
-public class TestS3AEncryptionAlgorithmPropagation extends AbstractS3ATestBase {
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- S3ATestUtils.disableFilesystemCaching(conf);
- conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
- "DES");
- return conf;
- }
-
- @Override
- public void teardown() throws Exception {
- super.teardown();
- IOUtils.closeStream(getFileSystem());
- }
-
- @Test
- public void testEncrypt0() throws Throwable {
- writeThenReadFileToFailure(0);
- }
-
- @Test
- public void testEncrypt256() throws Throwable {
- writeThenReadFileToFailure(256);
- }
-
- /**
- * Make this a no-op so test setup doesn't fail.
- * @param path path path
- * @throws IOException on any failure
- */
- @Override
- protected void mkdirs(Path path) throws IOException {
-
- }
-
- protected void writeThenReadFileToFailure(int len) throws IOException {
- skipIfEncryptionTestsDisabled(getConfiguration());
- describe("Create an encrypted file of size " + len);
- try {
- writeThenReadFile(methodName.getMethodName() + '-' + len, len);
- fail("Expected an exception about an illegal encryption algorithm");
- } catch (AWSS3IOException e) {
- assertStatusCode(e, 400);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionFastOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionFastOutputStream.java
deleted file mode 100644
index 1fa8486..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AEncryptionFastOutputStream.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Run the encryption tests against the Fast output stream.
- * This verifies that both file writing paths can encrypt their data.
- */
-public class TestS3AEncryptionFastOutputStream extends TestS3AEncryption {
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- conf.setBoolean(Constants.FAST_UPLOAD, true);
- return conf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java
deleted file mode 100644
index 58ac496..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFailureHandling.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.InvalidRequestException;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.nio.file.AccessDeniedException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
-
-/**
- * Test S3A Failure translation, including a functional test
- * generating errors during stream IO.
- */
-public class TestS3AFailureHandling extends AbstractFSContractTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AFailureHandling.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Test
- public void testReadFileChanged() throws Throwable {
- describe("overwrite a file with a shorter one during a read, seek");
- final int fullLength = 8192;
- final byte[] fullDataset = dataset(fullLength, 'a', 32);
- final int shortLen = 4096;
- final byte[] shortDataset = dataset(shortLen, 'A', 32);
- final FileSystem fs = getFileSystem();
- final Path testpath = path("readFileToChange.txt");
- // initial write
- writeDataset(fs, testpath, fullDataset, fullDataset.length, 1024, false);
- try(FSDataInputStream instream = fs.open(testpath)) {
- instream.seek(fullLength - 16);
- assertTrue("no data to read", instream.read() >= 0);
- // overwrite
- writeDataset(fs, testpath, shortDataset, shortDataset.length, 1024, true);
- // here the file length is less. Probe the file to see if this is true,
- // with a spin and wait
- eventually(30 *1000, new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- assertEquals(shortLen, fs.getFileStatus(testpath).getLen());
- return null;
- }
- });
- // here length is shorter. Assuming it has propagated to all replicas,
- // the position of the input stream is now beyond the EOF.
- // An attempt to seek backwards to a position greater than the
- // short length will raise an exception from AWS S3, which must be
- // translated into an EOF
-
- instream.seek(shortLen + 1024);
- int c = instream.read();
- assertIsEOF("read()", c);
-
- byte[] buf = new byte[256];
-
- assertIsEOF("read(buffer)", instream.read(buf));
- assertIsEOF("read(offset)",
- instream.read(instream.getPos(), buf, 0, buf.length));
-
- // now do a block read fully, again, backwards from the current pos
- try {
- instream.readFully(shortLen + 512, buf);
- fail("Expected readFully to fail");
- } catch (EOFException expected) {
- LOG.debug("Expected EOF: ", expected);
- }
-
- assertIsEOF("read(offset)",
- instream.read(shortLen + 510, buf, 0, buf.length));
-
- // seek somewhere useful
- instream.seek(shortLen - 256);
-
- // delete the file. Reads must fail
- fs.delete(testpath, false);
-
- try {
- int r = instream.read();
- fail("Expected an exception, got " + r);
- } catch (FileNotFoundException e) {
- // expected
- }
-
- try {
- instream.readFully(2048, buf);
- fail("Expected readFully to fail");
- } catch (FileNotFoundException e) {
- // expected
- }
-
- }
- }
-
- /**
- * Assert that a read operation returned an EOF value.
- * @param operation specific operation
- * @param readResult result
- */
- private void assertIsEOF(String operation, int readResult) {
- assertEquals("Expected EOF from "+ operation
- + "; got char " + (char) readResult, -1, readResult);
- }
-
- @Test
- public void test404isNotFound() throws Throwable {
- verifyTranslated(FileNotFoundException.class, createS3Exception(404));
- }
-
- protected Exception verifyTranslated(Class clazz,
- AmazonClientException exception) throws Exception {
- return verifyExceptionClass(clazz,
- translateException("test", "/", exception));
- }
-
- @Test
- public void test401isNotPermittedFound() throws Throwable {
- verifyTranslated(AccessDeniedException.class,
- createS3Exception(401));
- }
-
- protected AmazonS3Exception createS3Exception(int code) {
- AmazonS3Exception source = new AmazonS3Exception("");
- source.setStatusCode(code);
- return source;
- }
-
- @Test
- public void testGenericS3Exception() throws Throwable {
- // S3 exception of no known type
- AWSS3IOException ex = (AWSS3IOException)verifyTranslated(
- AWSS3IOException.class,
- createS3Exception(451));
- assertEquals(451, ex.getStatusCode());
- }
-
- @Test
- public void testGenericServiceS3Exception() throws Throwable {
- // service exception of no known type
- AmazonServiceException ase = new AmazonServiceException("unwind");
- ase.setStatusCode(500);
- AWSServiceIOException ex = (AWSServiceIOException)verifyTranslated(
- AWSServiceIOException.class,
- ase);
- assertEquals(500, ex.getStatusCode());
- }
-
- @Test
- public void testGenericClientException() throws Throwable {
- // Generic Amazon exception
- verifyTranslated(AWSClientIOException.class,
- new AmazonClientException(""));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java
deleted file mode 100644
index e507cf6..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFastOutputStream.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-/**
- * Tests regular and multi-part upload functionality for S3AFastOutputStream.
- * File sizes are kept small to reduce test duration on slow connections
- */
-public class TestS3AFastOutputStream {
- private FileSystem fs;
-
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024);
- conf.setInt(Constants.MULTIPART_SIZE, 5 * 1024 * 1024);
- conf.setBoolean(Constants.FAST_UPLOAD, true);
- fs = S3ATestUtils.createTestFileSystem(conf);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(getTestPath(), true);
- }
- }
-
- protected Path getTestPath() {
- return new Path("/tests3a");
- }
-
- @Test
- public void testRegularUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
- }
-
- @Test
- public void testMultiPartUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 *
- 1024);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java
deleted file mode 100644
index 0a8dd2d..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileOperationCost.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.net.URI;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.Statistic.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.MetricDiff;
-import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
-
-/**
- * Use metrics to assert about the cost of file status queries.
- * {@link S3AFileSystem#getFileStatus(Path)}.
- */
-public class TestS3AFileOperationCost extends AbstractFSContractTestBase {
-
- private MetricDiff metadataRequests;
- private MetricDiff listRequests;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AFileOperationCost.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Override
- public S3AFileSystem getFileSystem() {
- return (S3AFileSystem) super.getFileSystem();
- }
-
- @Override
- public void setup() throws Exception {
- super.setup();
- S3AFileSystem fs = getFileSystem();
- metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
- listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
- }
-
- @Test
- public void testCostOfGetFileStatusOnFile() throws Throwable {
- describe("performing getFileStatus on a file");
- Path simpleFile = path("simple.txt");
- S3AFileSystem fs = getFileSystem();
- touch(fs, simpleFile);
- resetMetricDiffs();
- S3AFileStatus status = fs.getFileStatus(simpleFile);
- assertTrue("not a file: " + status, status.isFile());
- metadataRequests.assertDiffEquals(1);
- listRequests.assertDiffEquals(0);
- }
-
- private void resetMetricDiffs() {
- reset(metadataRequests, listRequests);
- }
-
- @Test
- public void testCostOfGetFileStatusOnEmptyDir() throws Throwable {
- describe("performing getFileStatus on an empty directory");
- S3AFileSystem fs = getFileSystem();
- Path dir = path("empty");
- fs.mkdirs(dir);
- resetMetricDiffs();
- S3AFileStatus status = fs.getFileStatus(dir);
- assertTrue("not empty: " + status, status.isEmptyDirectory());
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(0);
- }
-
- @Test
- public void testCostOfGetFileStatusOnMissingFile() throws Throwable {
- describe("performing getFileStatus on a missing file");
- S3AFileSystem fs = getFileSystem();
- Path path = path("missing");
- resetMetricDiffs();
- try {
- S3AFileStatus status = fs.getFileStatus(path);
- fail("Got a status back from a missing file path " + status);
- } catch (FileNotFoundException expected) {
- // expected
- }
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(1);
- }
-
- @Test
- public void testCostOfGetFileStatusOnMissingSubPath() throws Throwable {
- describe("performing getFileStatus on a missing file");
- S3AFileSystem fs = getFileSystem();
- Path path = path("missingdir/missingpath");
- resetMetricDiffs();
- try {
- S3AFileStatus status = fs.getFileStatus(path);
- fail("Got a status back from a missing file path " + status);
- } catch (FileNotFoundException expected) {
- // expected
- }
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(1);
- }
-
- @Test
- public void testCostOfGetFileStatusOnNonEmptyDir() throws Throwable {
- describe("performing getFileStatus on a non-empty directory");
- S3AFileSystem fs = getFileSystem();
- Path dir = path("empty");
- fs.mkdirs(dir);
- Path simpleFile = new Path(dir, "simple.txt");
- touch(fs, simpleFile);
- resetMetricDiffs();
- S3AFileStatus status = fs.getFileStatus(dir);
- if (status.isEmptyDirectory()) {
- // erroneous state
- String fsState = fs.toString();
- fail("FileStatus says directory isempty: " + status
- + "\n" + ContractTestUtils.ls(fs, dir)
- + "\n" + fsState);
- }
- metadataRequests.assertDiffEquals(2);
- listRequests.assertDiffEquals(1);
- }
-
- @Test
- public void testCostOfCopyFromLocalFile() throws Throwable {
- describe("testCostOfCopyFromLocalFile");
- File localTestDir = getTestDir("tmp");
- localTestDir.mkdirs();
- File tmpFile = File.createTempFile("tests3acost", ".txt",
- localTestDir);
- tmpFile.delete();
- try {
- URI localFileURI = tmpFile.toURI();
- FileSystem localFS = FileSystem.get(localFileURI,
- getFileSystem().getConf());
- Path localPath = new Path(localFileURI);
- int len = 10 * 1024;
- byte[] data = dataset(len, 'A', 'Z');
- writeDataset(localFS, localPath, data, len, 1024, true);
- S3AFileSystem s3a = getFileSystem();
- MetricDiff copyLocalOps = new MetricDiff(s3a,
- INVOCATION_COPY_FROM_LOCAL_FILE);
- MetricDiff putRequests = new MetricDiff(s3a,
- OBJECT_PUT_REQUESTS);
- MetricDiff putBytes = new MetricDiff(s3a,
- OBJECT_PUT_BYTES);
-
- Path remotePath = path("copied");
- s3a.copyFromLocalFile(false, true, localPath, remotePath);
- verifyFileContents(s3a, remotePath, data);
- copyLocalOps.assertDiffEquals(1);
- putRequests.assertDiffEquals(1);
- putBytes.assertDiffEquals(len);
- // print final stats
- LOG.info("Filesystem {}", s3a);
- } finally {
- tmpFile.delete();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java
deleted file mode 100644
index 6315a80..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AFileSystemContract.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystemContractBaseTest;
-import org.apache.hadoop.fs.Path;
-
-/**
- * Tests a live S3 system. If your keys and bucket aren't specified, all tests
- * are marked as passed.
- *
- * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
- * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
- * properly making it impossible to skip the tests if we don't have a valid
- * bucket.
- **/
-public class TestS3AFileSystemContract extends FileSystemContractBaseTest {
-
- protected static final Logger LOG =
- LoggerFactory.getLogger(TestS3AFileSystemContract.class);
-
- @Override
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
-
- fs = S3ATestUtils.createTestFileSystem(conf);
- super.setUp();
- }
-
- @Override
- protected void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(path("test"), true);
- }
- super.tearDown();
- }
-
- @Override
- public void testMkdirsWithUmask() throws Exception {
- // not supported
- }
-
- @Override
- public void testRenameFileAsExistingFile() throws Exception {
- if (!renameSupported()) return;
-
- Path src = path("/test/hadoop/file");
- createFile(src);
- Path dst = path("/test/new/newfile");
- createFile(dst);
- // s3 doesn't support rename option
- // rename-overwrites-dest is always allowed.
- rename(src, dst, true, false, true);
- }
-
- @Override
- public void testRenameDirectoryAsExistingDirectory() throws Exception {
- if (!renameSupported()) {
- return;
- }
-
- Path src = path("/test/hadoop/dir");
- fs.mkdirs(src);
- createFile(path("/test/hadoop/dir/file1"));
- createFile(path("/test/hadoop/dir/subdir/file2"));
-
- Path dst = path("/test/new/newdir");
- fs.mkdirs(dst);
- rename(src, dst, true, false, true);
- assertFalse("Nested file1 exists",
- fs.exists(path("/test/hadoop/dir/file1")));
- assertFalse("Nested file2 exists",
- fs.exists(path("/test/hadoop/dir/subdir/file2")));
- assertTrue("Renamed nested file1 exists",
- fs.exists(path("/test/new/newdir/file1")));
- assertTrue("Renamed nested exists",
- fs.exists(path("/test/new/newdir/subdir/file2")));
- }
-
-// @Override
- public void testMoveDirUnderParent() throws Throwable {
- // not support because
- // Fails if dst is a directory that is not empty.
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
deleted file mode 100644
index 957dfb9..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import java.io.IOException;
-import java.net.URI;
-
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
-import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest;
-import com.amazonaws.services.securitytoken.model.GetSessionTokenResult;
-import com.amazonaws.services.securitytoken.model.Credentials;
-
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
-import org.apache.hadoop.fs.contract.s3a.S3AContract;
-import org.apache.hadoop.fs.s3native.S3xLoginHelper;
-import org.apache.hadoop.conf.Configuration;
-
-import org.junit.Test;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-
-/**
- * Tests use of temporary credentials (for example, AWS STS & S3).
- * This test extends a class that "does things to the root directory", and
- * should only be used against transient filesystems where you don't care about
- * the data.
- */
-public class TestS3ATemporaryCredentials extends AbstractFSContractTestBase {
- public static final String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled";
- public static final String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3ATemporaryCredentials.class);
-
- private static final String PROVIDER_CLASS
- = TemporaryAWSCredentialsProvider.NAME;
-
- private static final long TEST_FILE_SIZE = 1024;
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- /**
- * Test use of STS for requesting temporary credentials.
- *
- * The property test.sts.endpoint can be set to point this at different
- * STS endpoints. This test will use the AWS credentials (if provided) for
- * S3A tests to request temporary credentials, then attempt to use those
- * credentials instead.
- *
- * @throws IOException
- */
- @Test
- public void testSTS() throws IOException {
- Configuration conf = getContract().getConf();
- if (!conf.getBoolean(TEST_STS_ENABLED, true)) {
- skip("STS functional tests disabled");
- }
-
- S3xLoginHelper.Login login = S3AUtils.getAWSAccessKeys(
- URI.create("s3a://foobar"), conf);
- if (!login.hasLogin()) {
- skip("testSTS disabled because AWS credentials not configured");
- }
- AWSCredentialsProvider parentCredentials = new BasicAWSCredentialsProvider(
- login.getUser(), login.getPassword());
-
- String stsEndpoint = conf.getTrimmed(TEST_STS_ENDPOINT, "");
- AWSSecurityTokenServiceClient stsClient;
- stsClient = new AWSSecurityTokenServiceClient(parentCredentials);
- if (!stsEndpoint.isEmpty()) {
- LOG.debug("STS Endpoint ={}", stsEndpoint);
- stsClient.setEndpoint(stsEndpoint);
- }
- GetSessionTokenRequest sessionTokenRequest = new GetSessionTokenRequest();
- sessionTokenRequest.setDurationSeconds(900);
- GetSessionTokenResult sessionTokenResult;
- sessionTokenResult = stsClient.getSessionToken(sessionTokenRequest);
- Credentials sessionCreds = sessionTokenResult.getCredentials();
-
- String childAccessKey = sessionCreds.getAccessKeyId();
- conf.set(ACCESS_KEY, childAccessKey);
- String childSecretKey = sessionCreds.getSecretAccessKey();
- conf.set(SECRET_KEY, childSecretKey);
- String sessionToken = sessionCreds.getSessionToken();
- conf.set(SESSION_TOKEN, sessionToken);
-
- conf.set(AWS_CREDENTIALS_PROVIDER, PROVIDER_CLASS);
-
- try(S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
- createAndVerifyFile(fs, path("testSTS"), TEST_FILE_SIZE);
- }
-
- // now create an invalid set of credentials by changing the session
- // token
- conf.set(SESSION_TOKEN, "invalid-" + sessionToken);
- try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
- createAndVerifyFile(fs, path("testSTSInvalidToken"), TEST_FILE_SIZE);
- fail("Expected an access exception, but file access to "
- + fs.getUri() + " was allowed: " + fs);
- } catch (AWSS3IOException ex) {
- LOG.info("Expected Exception: {}", ex.toString());
- LOG.debug("Expected Exception: {}", ex, ex);
- }
- }
-
- @Test
- public void testTemporaryCredentialValidation() throws Throwable {
- Configuration conf = new Configuration();
- conf.set(ACCESS_KEY, "accesskey");
- conf.set(SECRET_KEY, "secretkey");
- conf.set(SESSION_TOKEN, "");
- TemporaryAWSCredentialsProvider provider
- = new TemporaryAWSCredentialsProvider(getFileSystem().getUri(), conf);
- try {
- AWSCredentials credentials = provider.getCredentials();
- fail("Expected a CredentialInitializationException,"
- + " got " + credentials);
- } catch (CredentialInitializationException expected) {
- // expected
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java
new file mode 100644
index 0000000..7e4273a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContext.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import org.apache.hadoop.fs.TestFileContext;
+
+/**
+ * Implementation of TestFileContext for S3a.
+ */
+public class ITestS3AFileContext extends TestFileContext{
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
new file mode 100644
index 0000000..4b8d4bb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+
+/**
+ * Extends FileContextCreateMkdirBaseTest for a S3a FileContext.
+ */
+public class ITestS3AFileContextCreateMkdir
+ extends FileContextCreateMkdirBaseTest {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
new file mode 100644
index 0000000..7247c16
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * S3A implementation of FileContextMainOperationsBaseTest.
+ */
+public class ITestS3AFileContextMainOperations
+ extends FileContextMainOperationsBaseTest {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+ @Override
+ protected boolean listCorruptedBlocksSupported() {
+ return false;
+ }
+
+ @Test
+ @Ignore
+ public void testCreateFlagAppendExistingFile() throws IOException {
+ //append not supported, so test removed
+ }
+
+ @Test
+ @Ignore
+ public void testCreateFlagCreateAppendExistingFile() throws IOException {
+ //append not supported, so test removed
+ }
+
+ @Test
+ @Ignore
+ public void testSetVerifyChecksum() throws IOException {
+ //checksums ignored, so test removed
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
new file mode 100644
index 0000000..5e88aba
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.net.URI;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FCStatisticsBaseTest;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+/**
+ * S3a implementation of FCStatisticsBaseTest.
+ */
+public class ITestS3AFileContextStatistics extends FCStatisticsBaseTest {
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"),
+ FileContext.DEFAULT_PERM, true);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true);
+ }
+
+ @Override
+ protected void verifyReadBytes(FileSystem.Statistics stats) {
+ // one blockSize for read, one for pread
+ Assert.assertEquals(2 * blockSize, stats.getBytesRead());
+ }
+
+ @Override
+ protected void verifyWrittenBytes(FileSystem.Statistics stats) {
+ //No extra bytes are written
+ Assert.assertEquals(blockSize, stats.getBytesWritten());
+ }
+
+ @Override
+ protected URI getFsUri() {
+ return fc.getHomeDirectory().toUri();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
new file mode 100644
index 0000000..fff1fcb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextURIBase;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * S3a implementation of FileContextURIBase.
+ */
+public class ITestS3AFileContextURI extends FileContextURIBase {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc1 = S3ATestUtils.createTestFileContext(conf);
+ fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
+ super.setUp();
+ }
+
+ @Test
+ @Ignore
+ public void testFileStatus() throws IOException {
+ // test disabled
+ // (the statistics tested with this method are not relevant for an S3FS)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java
new file mode 100644
index 0000000..d0312ba
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextUtil.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.fileContext;
+
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextUtilBase;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.junit.Before;
+
+/**
+ * S3A implementation of FileContextUtilBase.
+ */
+public class ITestS3AFileContextUtil extends FileContextUtilBase {
+
+ @Before
+ public void setUp() throws IOException, Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ super.setUp();
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/52] [abbrv] hadoop git commit: HDFS-10748.
TestFileTruncate#testTruncateWithDataNodesRestart runs sometimes timeout.
Contributed by Yiqun Lin.
Posted by aw...@apache.org.
HDFS-10748. TestFileTruncate#testTruncateWithDataNodesRestart runs sometimes timeout. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4da5000d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4da5000d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4da5000d
Branch: refs/heads/HADOOP-13341
Commit: 4da5000dd33cf013e7212848ed2c44f1e60e860e
Parents: 3d86110
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Aug 25 09:00:44 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Aug 25 09:00:44 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4da5000d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index 1032107..dd4dfbd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -700,6 +700,9 @@ public class TestFileTruncate {
assertEquals(newBlock.getBlock().getGenerationStamp(),
oldBlock.getBlock().getGenerationStamp() + 1);
+ Thread.sleep(2000);
+ // trigger the second time BR to delete the corrupted replica if there's one
+ cluster.triggerBlockReports();
// Wait replicas come to 3
DFSTestUtil.waitReplication(fs, p, REPLICATION);
// Old replica is disregarded and replaced with the truncated one
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/52] [abbrv] hadoop git commit: Revert "HDFS-10762. Pass IIP for
file status related methods. Contributed by Daryn Sharp."
Posted by aw...@apache.org.
Revert "HDFS-10762. Pass IIP for file status related methods. Contributed by Daryn Sharp."
This reverts commit 22fc46d7659972ff016ccf1c6f781f0c160be26f.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ca4d6dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ca4d6dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ca4d6dd
Branch: refs/heads/HADOOP-13341
Commit: 3ca4d6ddfd199c95677721ff3bcb95d1da45bd88
Parents: f4d4d34
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Aug 22 16:57:45 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Aug 22 16:57:45 2016 -0500
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirAppendOp.java | 6 +-
.../server/namenode/FSDirStatAndListingOp.java | 80 +++++++++++---------
.../hdfs/server/namenode/FSDirWriteFileOp.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 14 ++--
.../hdfs/server/namenode/INodesInPath.java | 42 ++--------
.../hadoop/hdfs/TestReservedRawPaths.java | 21 -----
6 files changed, 64 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 5192352..3a5d7dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -85,10 +85,9 @@ final class FSDirAppendOp {
final LocatedBlock lb;
final FSDirectory fsd = fsn.getFSDirectory();
final String src;
- final INodesInPath iip;
fsd.writeLock();
try {
- iip = fsd.resolvePathForWrite(pc, srcArg);
+ final INodesInPath iip = fsd.resolvePathForWrite(pc, srcArg);
src = iip.getPath();
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
@@ -149,7 +148,8 @@ final class FSDirAppendOp {
fsd.writeUnlock();
}
- HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, iip);
+ HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, src, false,
+ FSDirectory.isReservedRawName(srcArg));
if (lb != null) {
NameNode.stateChangeLog.debug(
"DIR* NameSystem.appendFile: file {} for {} at {} block {} block"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 88be510..c9eedf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -108,16 +108,16 @@ class FSDirStatAndListingOp {
if (!DFSUtil.isValidName(src)) {
throw new InvalidPathException("Invalid file name: " + src);
}
- final INodesInPath iip;
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- iip = fsd.resolvePath(pc, srcArg, resolveLink);
+ final INodesInPath iip = fsd.resolvePath(pc, srcArg, resolveLink);
+ src = iip.getPath();
fsd.checkPermission(pc, iip, false, null, null, null, null, false);
} else {
src = FSDirectory.resolvePath(srcArg, fsd);
- iip = fsd.getINodesInPath(src, resolveLink);
}
- return getFileInfo(fsd, iip);
+ return getFileInfo(fsd, src, FSDirectory.isReservedRawName(srcArg),
+ resolveLink);
}
/**
@@ -230,6 +230,7 @@ class FSDirStatAndListingOp {
String src, byte[] startAfter, boolean needLocation, boolean isSuperUser)
throws IOException {
String srcs = FSDirectory.normalizePath(src);
+ final boolean isRawPath = FSDirectory.isReservedRawName(src);
if (FSDirectory.isExactReservedName(srcs)) {
return getReservedListing(fsd);
}
@@ -256,7 +257,7 @@ class FSDirStatAndListingOp {
return new DirectoryListing(
new HdfsFileStatus[]{ createFileStatus(
fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- needLocation, parentStoragePolicy, iip)
+ needLocation, parentStoragePolicy, snapshot, isRawPath, iip)
}, 0);
}
@@ -281,7 +282,7 @@ class FSDirStatAndListingOp {
cur.getLocalNameBytes());
listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), nodeAttrs,
needLocation, getStoragePolicyID(curPolicy, parentStoragePolicy),
- iipWithChild);
+ snapshot, isRawPath, iipWithChild);
listingCnt++;
if (needLocation) {
// Once we hit lsLimit locations, stop.
@@ -338,6 +339,7 @@ class FSDirStatAndListingOp {
listing[i] = createFileStatus(
fsd, sRoot.getLocalNameBytes(), nodeAttrs,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
+ Snapshot.CURRENT_STATE_ID, false,
INodesInPath.fromINode(sRoot));
}
return new DirectoryListing(
@@ -361,8 +363,10 @@ class FSDirStatAndListingOp {
* @return object containing information regarding the file
* or null if file not found
*/
- static HdfsFileStatus getFileInfo(FSDirectory fsd,
- INodesInPath iip, boolean includeStoragePolicy) throws IOException {
+ static HdfsFileStatus getFileInfo(
+ FSDirectory fsd, String path, INodesInPath iip, boolean isRawPath,
+ boolean includeStoragePolicy)
+ throws IOException {
fsd.readLock();
try {
final INode node = iip.getLastINode();
@@ -373,32 +377,36 @@ class FSDirStatAndListingOp {
byte policyId = includeStoragePolicy && !node.isSymlink() ?
node.getStoragePolicyID() :
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
- INodeAttributes nodeAttrs = getINodeAttributes(fsd, iip.getPath(),
+ INodeAttributes nodeAttrs = getINodeAttributes(fsd, path,
HdfsFileStatus.EMPTY_NAME,
node, iip.getPathSnapshotId());
return createFileStatus(fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- policyId, iip);
+ policyId, iip.getPathSnapshotId(), isRawPath, iip);
} finally {
fsd.readUnlock();
}
}
- static HdfsFileStatus getFileInfo(FSDirectory fsd, INodesInPath iip)
+ static HdfsFileStatus getFileInfo(
+ FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath)
throws IOException {
- if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
- return FSDirectory.DOT_RESERVED_STATUS;
- }
-
- if (iip.isDotSnapshotDir()) {
- if (fsd.getINode4DotSnapshot(iip) != null) {
- return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
- HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
- HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
+ fsd.readLock();
+ try {
+ HdfsFileStatus status = null;
+ final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
+ if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
+ status = FSDirectory.DOT_RESERVED_STATUS;
+ } else if (iip.isDotSnapshotDir()) {
+ if (fsd.getINode4DotSnapshot(iip) != null) {
+ status = FSDirectory.DOT_SNAPSHOT_DIR_STATUS;
+ }
+ } else {
+ status = getFileInfo(fsd, src, iip, isRawPath, true);
}
- return null;
+ return status;
+ } finally {
+ fsd.readUnlock();
}
-
- return getFileInfo(fsd, iip, true);
}
/**
@@ -415,12 +423,15 @@ class FSDirStatAndListingOp {
*/
private static HdfsFileStatus createFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- boolean needLocation, byte storagePolicy, INodesInPath iip)
+ boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath,
+ INodesInPath iip)
throws IOException {
if (needLocation) {
- return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
+ return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy,
+ snapshot, isRawPath, iip);
} else {
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
+ snapshot, isRawPath, iip);
}
}
@@ -434,7 +445,8 @@ class FSDirStatAndListingOp {
INodesInPath iip) throws IOException {
INodeAttributes nodeAttrs = getINodeAttributes(
fsd, fullPath, path, iip.getLastINode(), snapshot);
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
+ snapshot, isRawPath, iip);
}
/**
@@ -442,15 +454,14 @@ class FSDirStatAndListingOp {
* @param iip the INodesInPath containing the target INode and its ancestors
*/
static HdfsFileStatus createFileStatus(
- FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- byte storagePolicy, INodesInPath iip) throws IOException {
+ FSDirectory fsd, byte[] path,
+ INodeAttributes nodeAttrs, byte storagePolicy, int snapshot,
+ boolean isRawPath, INodesInPath iip) throws IOException {
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
final boolean isEncrypted;
final INode node = iip.getLastINode();
- final int snapshot = iip.getPathSnapshotId();
- final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
@@ -500,9 +511,10 @@ class FSDirStatAndListingOp {
* Create FileStatus with location info by file INode
* @param iip the INodesInPath containing the target INode and its ancestors
*/
- private static HdfsFileStatus createLocatedFileStatus(
+ private static HdfsLocatedFileStatus createLocatedFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- byte storagePolicy, INodesInPath iip) throws IOException {
+ byte storagePolicy, int snapshot,
+ boolean isRawPath, INodesInPath iip) throws IOException {
assert fsd.hasReadLock();
long size = 0; // length is zero for directories
short replication = 0;
@@ -510,8 +522,6 @@ class FSDirStatAndListingOp {
LocatedBlocks loc = null;
final boolean isEncrypted;
final INode node = iip.getLastINode();
- final int snapshot = iip.getPathSnapshotId();
- final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 030d8cc..8429024 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -340,6 +340,7 @@ class FSDirWriteFileOp {
version = ezInfo.protocolVersion;
}
+ boolean isRawPath = FSDirectory.isReservedRawName(src);
FSDirectory fsd = fsn.getFSDirectory();
INodesInPath iip = fsd.resolvePathForWrite(pc, src);
src = iip.getPath();
@@ -443,7 +444,7 @@ class FSDirWriteFileOp {
NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
src + " inode " + newNode.getId() + " " + holder);
}
- return FSDirStatAndListingOp.getFileInfo(fsd, iip);
+ return FSDirStatAndListingOp.getFileInfo(fsd, src, false, isRawPath);
}
static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 5dc5a03..b6652e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -531,24 +531,21 @@ public class FSDirectory implements Closeable {
* @throws FileNotFoundException
* @throws AccessControlException
*/
- @VisibleForTesting
- public INodesInPath resolvePath(FSPermissionChecker pc, String src)
+ INodesInPath resolvePath(FSPermissionChecker pc, String src)
throws UnresolvedLinkException, FileNotFoundException,
AccessControlException {
return resolvePath(pc, src, true);
}
- @VisibleForTesting
- public INodesInPath resolvePath(FSPermissionChecker pc, String src,
+ INodesInPath resolvePath(FSPermissionChecker pc, String src,
boolean resolveLink) throws UnresolvedLinkException,
FileNotFoundException, AccessControlException {
byte[][] components = INode.getPathComponents(src);
- boolean isRaw = isReservedRawName(components);
- if (isPermissionEnabled && pc != null && isRaw) {
+ if (isPermissionEnabled && pc != null && isReservedRawName(components)) {
pc.checkSuperuserPrivilege();
}
components = resolveComponents(components, this);
- return INodesInPath.resolve(rootDir, components, isRaw, resolveLink);
+ return INodesInPath.resolve(rootDir, components, resolveLink);
}
INodesInPath resolvePathForWrite(FSPermissionChecker pc, String src)
@@ -1665,7 +1662,8 @@ public class FSDirectory implements Closeable {
HdfsFileStatus getAuditFileInfo(INodesInPath iip)
throws IOException {
return (namesystem.isAuditEnabled() && namesystem.isExternalInvocation())
- ? FSDirStatAndListingOp.getFileInfo(this, iip, false) : null;
+ ? FSDirStatAndListingOp.getFileInfo(this, iip.getPath(), iip, false,
+ false) : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index af8998f..86cab28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -126,12 +126,6 @@ public class INodesInPath {
static INodesInPath resolve(final INodeDirectory startingDir,
final byte[][] components, final boolean resolveLink)
throws UnresolvedLinkException {
- return resolve(startingDir, components, false, resolveLink);
- }
-
- static INodesInPath resolve(final INodeDirectory startingDir,
- final byte[][] components, final boolean isRaw,
- final boolean resolveLink) throws UnresolvedLinkException {
Preconditions.checkArgument(startingDir.compareTo(components[0]) == 0);
INode curNode = startingDir;
@@ -231,7 +225,7 @@ public class INodesInPath {
System.arraycopy(inodes, 0, newNodes, 0, newNodes.length);
inodes = newNodes;
}
- return new INodesInPath(inodes, components, isRaw, isSnapshot, snapshotId);
+ return new INodesInPath(inodes, components, isSnapshot, snapshotId);
}
private static boolean shouldUpdateLatestId(int sid, int snapshotId) {
@@ -255,8 +249,7 @@ public class INodesInPath {
INode[] inodes = new INode[iip.inodes.length];
System.arraycopy(iip.inodes, 0, inodes, 0, inodes.length);
inodes[pos] = inode;
- return new INodesInPath(inodes, iip.path, iip.isRaw,
- iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, iip.path, iip.isSnapshot, iip.snapshotId);
}
/**
@@ -274,8 +267,7 @@ public class INodesInPath {
byte[][] path = new byte[iip.path.length + 1][];
System.arraycopy(iip.path, 0, path, 0, path.length - 1);
path[path.length - 1] = childName;
- return new INodesInPath(inodes, path, iip.isRaw,
- iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, path, iip.isSnapshot, iip.snapshotId);
}
private final byte[][] path;
@@ -287,13 +279,6 @@ public class INodesInPath {
* true if this path corresponds to a snapshot
*/
private final boolean isSnapshot;
-
- /**
- * true if this is a /.reserved/raw path. path component resolution strips
- * it from the path so need to track it separately.
- */
- private final boolean isRaw;
-
/**
* For snapshot paths, it is the id of the snapshot; or
* {@link Snapshot#CURRENT_STATE_ID} if the snapshot does not exist. For
@@ -302,18 +287,17 @@ public class INodesInPath {
*/
private final int snapshotId;
- private INodesInPath(INode[] inodes, byte[][] path, boolean isRaw,
- boolean isSnapshot,int snapshotId) {
+ private INodesInPath(INode[] inodes, byte[][] path, boolean isSnapshot,
+ int snapshotId) {
Preconditions.checkArgument(inodes != null && path != null);
this.inodes = inodes;
this.path = path;
- this.isRaw = isRaw;
this.isSnapshot = isSnapshot;
this.snapshotId = snapshotId;
}
private INodesInPath(INode[] inodes, byte[][] path) {
- this(inodes, path, false, false, CURRENT_STATE_ID);
+ this(inodes, path, false, CURRENT_STATE_ID);
}
/**
@@ -416,7 +400,7 @@ public class INodesInPath {
final byte[][] apath = new byte[length][];
System.arraycopy(this.inodes, 0, anodes, 0, length);
System.arraycopy(this.path, 0, apath, 0, length);
- return new INodesInPath(anodes, apath, isRaw, false, snapshotId);
+ return new INodesInPath(anodes, apath, false, snapshotId);
}
/**
@@ -444,7 +428,7 @@ public class INodesInPath {
byte[][] existingPath = new byte[i][];
System.arraycopy(inodes, 0, existing, 0, i);
System.arraycopy(path, 0, existingPath, 0, i);
- return new INodesInPath(existing, existingPath, isRaw, false, snapshotId);
+ return new INodesInPath(existing, existingPath, false, snapshotId);
}
/**
@@ -454,20 +438,10 @@ public class INodesInPath {
return this.isSnapshot;
}
- /**
- * @return if .snapshot is the last path component.
- */
boolean isDotSnapshotDir() {
return isDotSnapshotDir(getLastLocalName());
}
- /**
- * @return if this is a /.reserved/raw path.
- */
- public boolean isRaw() {
- return isRaw;
- }
-
private static String toString(INode inode) {
return inode == null? null: inode.getLocalName();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ca4d6dd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
index c09d346..449f715 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
-import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
@@ -50,8 +49,6 @@ import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesNotEqual;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.apache.hadoop.test.GenericTestUtils.assertMatches;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestReservedRawPaths {
@@ -103,24 +100,6 @@ public class TestReservedRawPaths {
}
/**
- * Verify resolving path will return an iip that tracks if the original
- * path was a raw path.
- */
- @Test(timeout = 120000)
- public void testINodesInPath() throws IOException {
- FSDirectory fsd = cluster.getNamesystem().getFSDirectory();
- final String path = "/path";
-
- INodesInPath iip = fsd.resolvePath(null, path);
- assertFalse(iip.isRaw());
- assertEquals(path, iip.getPath());
-
- iip = fsd.resolvePath(null, "/.reserved/raw" + path);
- assertTrue(iip.isRaw());
- assertEquals(path, iip.getPath());
- }
-
- /**
* Basic read/write tests of raw files.
* Create a non-encrypted file
* Create an encryption zone
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/52] [abbrv] hadoop git commit: MAPREDUCE-6767. TestSlive fails
after a common change. Contributed by Daniel Templeton.
Posted by aw...@apache.org.
MAPREDUCE-6767. TestSlive fails after a common change. Contributed by Daniel Templeton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a6fc5f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a6fc5f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a6fc5f4
Branch: refs/heads/HADOOP-13341
Commit: 5a6fc5f4831d130d9bdaca03f02f0ac4e7cfd64f
Parents: a1f3293
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Aug 25 10:55:00 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Aug 25 10:55:00 2016 +0900
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/fs/slive/AppendOp.java | 2 +-
.../src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a6fc5f4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java
index 3d9ba1e..82d221f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java
@@ -101,7 +101,7 @@ class AppendOp extends Operation {
out.add(new OperationOutput(OutputType.LONG, getType(),
ReportWriter.NOT_FOUND, 1L));
LOG.warn("Error with appending", e);
- } catch (IOException e) {
+ } catch (IOException | UnsupportedOperationException e) {
out.add(new OperationOutput(OutputType.LONG, getType(),
ReportWriter.FAILURES, 1L));
LOG.warn("Error with appending", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a6fc5f4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java
index c845ac1..202d807 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java
@@ -94,7 +94,7 @@ class TruncateOp extends Operation {
out.add(new OperationOutput(OutputType.LONG, getType(),
ReportWriter.NOT_FOUND, 1L));
LOG.warn("Error with truncating", e);
- } catch (IOException e) {
+ } catch (IOException | UnsupportedOperationException e) {
out.add(new OperationOutput(OutputType.LONG, getType(),
ReportWriter.FAILURES, 1L));
LOG.warn("Error with truncating", e);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/52] [abbrv] hadoop git commit: HADOOP-13465. Design Server.Call
to be extensible for unified call queue. Contributed by Daryn Sharp.
Posted by aw...@apache.org.
HADOOP-13465. Design Server.Call to be extensible for unified call queue. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d288a0ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d288a0ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d288a0ba
Branch: refs/heads/HADOOP-13341
Commit: d288a0ba8364d81aacda9f4a21022eecb6dc4e22
Parents: 4da5000
Author: Kihwal Lee <ki...@apache.org>
Authored: Thu Aug 25 11:43:39 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Thu Aug 25 11:44:13 2016 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Server.java | 336 +++++++++++--------
1 file changed, 191 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d288a0ba/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 4c73f6a..09fe889 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -354,10 +354,9 @@ public abstract class Server {
*/
public static InetAddress getRemoteIp() {
Call call = CurCall.get();
- return (call != null && call.connection != null) ? call.connection
- .getHostInetAddress() : null;
+ return (call != null ) ? call.getHostInetAddress() : null;
}
-
+
/**
* Returns the clientId from the current RPC request
*/
@@ -380,10 +379,9 @@ public abstract class Server {
*/
public static UserGroupInformation getRemoteUser() {
Call call = CurCall.get();
- return (call != null && call.connection != null) ? call.connection.user
- : null;
+ return (call != null) ? call.getRemoteUser() : null;
}
-
+
/** Return true if the invocation was through an RPC.
*/
public static boolean isRpcInvocation() {
@@ -483,7 +481,7 @@ public abstract class Server {
if ((rpcMetrics.getProcessingSampleCount() > minSampleSize) &&
(processingTime > threeSigma)) {
if(LOG.isWarnEnabled()) {
- String client = CurCall.get().connection.toString();
+ String client = CurCall.get().toString();
LOG.warn(
"Slow RPC : " + methodName + " took " + processingTime +
" milliseconds to process from client " + client);
@@ -657,62 +655,65 @@ public abstract class Server {
CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT);
}
- /** A call queued for handling. */
- public static class Call implements Schedulable {
- private final int callId; // the client's call id
- private final int retryCount; // the retry count of the call
- private final Writable rpcRequest; // Serialized Rpc request from client
- private final Connection connection; // connection to client
- private long timestamp; // time received when response is null
- // time served when response is not null
- private ByteBuffer rpcResponse; // the response for this call
+ /** A generic call queued for handling. */
+ public static class Call implements Schedulable,
+ PrivilegedExceptionAction<Void> {
+ final int callId; // the client's call id
+ final int retryCount; // the retry count of the call
+ long timestamp; // time received when response is null
+ // time served when response is not null
private AtomicInteger responseWaitCount = new AtomicInteger(1);
- private final RPC.RpcKind rpcKind;
- private final byte[] clientId;
+ final RPC.RpcKind rpcKind;
+ final byte[] clientId;
private final TraceScope traceScope; // the HTrace scope on the server side
private final CallerContext callerContext; // the call context
private int priorityLevel;
// the priority level assigned by scheduler, 0 by default
- private Call(Call call) {
- this(call.callId, call.retryCount, call.rpcRequest, call.connection,
- call.rpcKind, call.clientId, call.traceScope, call.callerContext);
+ Call(Call call) {
+ this(call.callId, call.retryCount, call.rpcKind, call.clientId,
+ call.traceScope, call.callerContext);
}
- public Call(int id, int retryCount, Writable param,
- Connection connection) {
- this(id, retryCount, param, connection, RPC.RpcKind.RPC_BUILTIN,
- RpcConstants.DUMMY_CLIENT_ID);
+ Call(int id, int retryCount, RPC.RpcKind kind, byte[] clientId) {
+ this(id, retryCount, kind, clientId, null, null);
}
- public Call(int id, int retryCount, Writable param, Connection connection,
+ @VisibleForTesting // primarily TestNamenodeRetryCache
+ public Call(int id, int retryCount, Void ignore1, Void ignore2,
RPC.RpcKind kind, byte[] clientId) {
- this(id, retryCount, param, connection, kind, clientId, null, null);
+ this(id, retryCount, kind, clientId, null, null);
}
- public Call(int id, int retryCount, Writable param, Connection connection,
- RPC.RpcKind kind, byte[] clientId, TraceScope traceScope,
- CallerContext callerContext) {
+ Call(int id, int retryCount, RPC.RpcKind kind, byte[] clientId,
+ TraceScope traceScope, CallerContext callerContext) {
this.callId = id;
this.retryCount = retryCount;
- this.rpcRequest = param;
- this.connection = connection;
this.timestamp = Time.now();
- this.rpcResponse = null;
this.rpcKind = kind;
this.clientId = clientId;
this.traceScope = traceScope;
this.callerContext = callerContext;
}
-
+
@Override
public String toString() {
- return rpcRequest + " from " + connection + " Call#" + callId + " Retry#"
- + retryCount;
+ return "Call#" + callId + " Retry#" + retryCount;
}
- public void setResponse(ByteBuffer response) {
- this.rpcResponse = response;
+ public Void run() throws Exception {
+ return null;
+ }
+ // should eventually be abstract but need to avoid breaking tests
+ public UserGroupInformation getRemoteUser() {
+ return null;
+ }
+ public InetAddress getHostInetAddress() {
+ return null;
+ }
+ public String getHostAddress() {
+ InetAddress addr = getHostInetAddress();
+ return (addr != null) ? addr.getHostAddress() : null;
}
/**
@@ -724,34 +725,36 @@ public abstract class Server {
*/
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public void postponeResponse() {
+ public final void postponeResponse() {
int count = responseWaitCount.incrementAndGet();
assert count > 0 : "response has already been sent";
}
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public void sendResponse() throws IOException {
+ public final void sendResponse() throws IOException {
int count = responseWaitCount.decrementAndGet();
assert count >= 0 : "response has already been sent";
if (count == 0) {
- connection.sendResponse(this);
+ doResponse(null);
}
}
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public void abortResponse(Throwable t) throws IOException {
+ public final void abortResponse(Throwable t) throws IOException {
// don't send response if the call was already sent or aborted.
if (responseWaitCount.getAndSet(-1) > 0) {
- connection.abortResponse(this, t);
+ doResponse(t);
}
}
+ void doResponse(Throwable t) throws IOException {}
+
// For Schedulable
@Override
public UserGroupInformation getUserGroupInformation() {
- return connection.user;
+ return getRemoteUser();
}
@Override
@@ -764,6 +767,114 @@ public abstract class Server {
}
}
+ /** A RPC extended call queued for handling. */
+ private class RpcCall extends Call {
+ final Connection connection; // connection to client
+ final Writable rpcRequest; // Serialized Rpc request from client
+ ByteBuffer rpcResponse; // the response for this call
+
+ RpcCall(RpcCall call) {
+ super(call);
+ this.connection = call.connection;
+ this.rpcRequest = call.rpcRequest;
+ }
+
+ RpcCall(Connection connection, int id) {
+ this(connection, id, RpcConstants.INVALID_RETRY_COUNT);
+ }
+
+ RpcCall(Connection connection, int id, int retryCount) {
+ this(connection, id, retryCount, null,
+ RPC.RpcKind.RPC_BUILTIN, RpcConstants.DUMMY_CLIENT_ID,
+ null, null);
+ }
+
+ RpcCall(Connection connection, int id, int retryCount,
+ Writable param, RPC.RpcKind kind, byte[] clientId,
+ TraceScope traceScope, CallerContext context) {
+ super(id, retryCount, kind, clientId, traceScope, context);
+ this.connection = connection;
+ this.rpcRequest = param;
+ }
+
+ @Override
+ public UserGroupInformation getRemoteUser() {
+ return connection.user;
+ }
+
+ @Override
+ public InetAddress getHostInetAddress() {
+ return connection.getHostInetAddress();
+ }
+
+ @Override
+ public Void run() throws Exception {
+ if (!connection.channel.isOpen()) {
+ Server.LOG.info(Thread.currentThread().getName() + ": skipped " + this);
+ return null;
+ }
+ String errorClass = null;
+ String error = null;
+ RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
+ RpcErrorCodeProto detailedErr = null;
+ Writable value = null;
+
+ try {
+ value = call(
+ rpcKind, connection.protocolName, rpcRequest, timestamp);
+ } catch (Throwable e) {
+ if (e instanceof UndeclaredThrowableException) {
+ e = e.getCause();
+ }
+ logException(Server.LOG, e, this);
+ if (e instanceof RpcServerException) {
+ RpcServerException rse = ((RpcServerException)e);
+ returnStatus = rse.getRpcStatusProto();
+ detailedErr = rse.getRpcErrorCodeProto();
+ } else {
+ returnStatus = RpcStatusProto.ERROR;
+ detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
+ }
+ errorClass = e.getClass().getName();
+ error = StringUtils.stringifyException(e);
+ // Remove redundant error class name from the beginning of the
+ // stack trace
+ String exceptionHdr = errorClass + ": ";
+ if (error.startsWith(exceptionHdr)) {
+ error = error.substring(exceptionHdr.length());
+ }
+ }
+ setupResponse(this, returnStatus, detailedErr,
+ value, errorClass, error);
+ sendResponse();
+ return null;
+ }
+
+ void setResponse(ByteBuffer response) throws IOException {
+ this.rpcResponse = response;
+ }
+
+ @Override
+ void doResponse(Throwable t) throws IOException {
+ RpcCall call = this;
+ if (t != null) {
+ // clone the call to prevent a race with another thread stomping
+ // on the response while being sent. the original call is
+ // effectively discarded since the wait count won't hit zero
+ call = new RpcCall(this);
+ setupResponse(call,
+ RpcStatusProto.FATAL, RpcErrorCodeProto.ERROR_RPC_SERVER,
+ null, t.getClass().getName(), StringUtils.stringifyException(t));
+ }
+ connection.sendResponse(this);
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " " + rpcRequest + " from " + connection;
+ }
+ }
+
/** Listens on the socket. Creates jobs for the handler threads*/
private class Listener extends Thread {
@@ -1094,22 +1205,22 @@ public abstract class Server {
if(LOG.isDebugEnabled()) {
LOG.debug("Checking for old call responses.");
}
- ArrayList<Call> calls;
+ ArrayList<RpcCall> calls;
// get the list of channels from list of keys.
synchronized (writeSelector.keys()) {
- calls = new ArrayList<Call>(writeSelector.keys().size());
+ calls = new ArrayList<RpcCall>(writeSelector.keys().size());
iter = writeSelector.keys().iterator();
while (iter.hasNext()) {
SelectionKey key = iter.next();
- Call call = (Call)key.attachment();
+ RpcCall call = (RpcCall)key.attachment();
if (call != null && key.channel() == call.connection.channel) {
calls.add(call);
}
}
}
-
- for(Call call : calls) {
+
+ for (RpcCall call : calls) {
doPurge(call, now);
}
} catch (OutOfMemoryError e) {
@@ -1127,7 +1238,7 @@ public abstract class Server {
}
private void doAsyncWrite(SelectionKey key) throws IOException {
- Call call = (Call)key.attachment();
+ RpcCall call = (RpcCall)key.attachment();
if (call == null) {
return;
}
@@ -1155,10 +1266,10 @@ public abstract class Server {
// Remove calls that have been pending in the responseQueue
// for a long time.
//
- private void doPurge(Call call, long now) {
- LinkedList<Call> responseQueue = call.connection.responseQueue;
+ private void doPurge(RpcCall call, long now) {
+ LinkedList<RpcCall> responseQueue = call.connection.responseQueue;
synchronized (responseQueue) {
- Iterator<Call> iter = responseQueue.listIterator(0);
+ Iterator<RpcCall> iter = responseQueue.listIterator(0);
while (iter.hasNext()) {
call = iter.next();
if (now > call.timestamp + PURGE_INTERVAL) {
@@ -1172,12 +1283,12 @@ public abstract class Server {
// Processes one response. Returns true if there are no more pending
// data for this channel.
//
- private boolean processResponse(LinkedList<Call> responseQueue,
+ private boolean processResponse(LinkedList<RpcCall> responseQueue,
boolean inHandler) throws IOException {
boolean error = true;
boolean done = false; // there is more data for this channel.
int numElements = 0;
- Call call = null;
+ RpcCall call = null;
try {
synchronized (responseQueue) {
//
@@ -1260,7 +1371,7 @@ public abstract class Server {
//
// Enqueue a response from the application.
//
- void doRespond(Call call) throws IOException {
+ void doRespond(RpcCall call) throws IOException {
synchronized (call.connection.responseQueue) {
// must only wrap before adding to the responseQueue to prevent
// postponed responses from being encrypted and sent out of order.
@@ -1358,7 +1469,7 @@ public abstract class Server {
private SocketChannel channel;
private ByteBuffer data;
private ByteBuffer dataLengthBuffer;
- private LinkedList<Call> responseQueue;
+ private LinkedList<RpcCall> responseQueue;
// number of outstanding rpcs
private AtomicInteger rpcCount = new AtomicInteger();
private long lastContact;
@@ -1385,8 +1496,8 @@ public abstract class Server {
public UserGroupInformation attemptingUser = null; // user name before auth
// Fake 'call' for failed authorization response
- private final Call authFailedCall = new Call(AUTHORIZATION_FAILED_CALL_ID,
- RpcConstants.INVALID_RETRY_COUNT, null, this);
+ private final RpcCall authFailedCall =
+ new RpcCall(this, AUTHORIZATION_FAILED_CALL_ID);
private boolean sentNegotiate = false;
private boolean useWrap = false;
@@ -1409,7 +1520,7 @@ public abstract class Server {
this.hostAddress = addr.getHostAddress();
}
this.remotePort = socket.getPort();
- this.responseQueue = new LinkedList<Call>();
+ this.responseQueue = new LinkedList<RpcCall>();
if (socketSendBufferSize != 0) {
try {
socket.setSendBufferSize(socketSendBufferSize);
@@ -1704,8 +1815,7 @@ public abstract class Server {
}
private void doSaslReply(Message message) throws IOException {
- final Call saslCall = new Call(AuthProtocol.SASL.callId,
- RpcConstants.INVALID_RETRY_COUNT, null, this);
+ final RpcCall saslCall = new RpcCall(this, AuthProtocol.SASL.callId);
setupResponse(saslCall,
RpcStatusProto.SUCCESS, null,
RpcWritable.wrap(message), null, null);
@@ -1922,23 +2032,20 @@ public abstract class Server {
if (clientVersion >= 9) {
// Versions >>9 understand the normal response
- Call fakeCall = new Call(-1, RpcConstants.INVALID_RETRY_COUNT, null,
- this);
+ RpcCall fakeCall = new RpcCall(this, -1);
setupResponse(fakeCall,
RpcStatusProto.FATAL, RpcErrorCodeProto.FATAL_VERSION_MISMATCH,
null, VersionMismatch.class.getName(), errMsg);
fakeCall.sendResponse();
} else if (clientVersion >= 3) {
- Call fakeCall = new Call(-1, RpcConstants.INVALID_RETRY_COUNT, null,
- this);
+ RpcCall fakeCall = new RpcCall(this, -1);
// Versions 3 to 8 use older response
setupResponseOldVersionFatal(buffer, fakeCall,
null, VersionMismatch.class.getName(), errMsg);
fakeCall.sendResponse();
} else if (clientVersion == 2) { // Hadoop 0.18.3
- Call fakeCall = new Call(0, RpcConstants.INVALID_RETRY_COUNT, null,
- this);
+ RpcCall fakeCall = new RpcCall(this, 0);
DataOutputStream out = new DataOutputStream(buffer);
out.writeInt(0); // call ID
out.writeBoolean(true); // error
@@ -1950,7 +2057,7 @@ public abstract class Server {
}
private void setupHttpRequestOnIpcPortResponse() throws IOException {
- Call fakeCall = new Call(0, RpcConstants.INVALID_RETRY_COUNT, null, this);
+ RpcCall fakeCall = new RpcCall(this, 0);
fakeCall.setResponse(ByteBuffer.wrap(
RECEIVED_HTTP_REQ_RESPONSE.getBytes(StandardCharsets.UTF_8)));
fakeCall.sendResponse();
@@ -2098,7 +2205,7 @@ public abstract class Server {
}
} catch (WrappedRpcServerException wrse) { // inform client of error
Throwable ioe = wrse.getCause();
- final Call call = new Call(callId, retry, null, this);
+ final RpcCall call = new RpcCall(this, callId, retry);
setupResponse(call,
RpcStatusProto.FATAL, wrse.getRpcErrorCodeProto(), null,
ioe.getClass().getName(), ioe.getMessage());
@@ -2198,8 +2305,9 @@ public abstract class Server {
.build();
}
- Call call = new Call(header.getCallId(), header.getRetryCount(),
- rpcRequest, this, ProtoUtil.convert(header.getRpcKind()),
+ RpcCall call = new RpcCall(this, header.getCallId(),
+ header.getRetryCount(), rpcRequest,
+ ProtoUtil.convert(header.getRpcKind()),
header.getClientId().toByteArray(), traceScope, callerContext);
// Save the priority level assignment by the scheduler
@@ -2323,21 +2431,10 @@ public abstract class Server {
}
}
- private void sendResponse(Call call) throws IOException {
+ private void sendResponse(RpcCall call) throws IOException {
responder.doRespond(call);
}
- private void abortResponse(Call call, Throwable t) throws IOException {
- // clone the call to prevent a race with the other thread stomping
- // on the response while being sent. the original call is
- // effectively discarded since the wait count won't hit zero
- call = new Call(call);
- setupResponse(call,
- RpcStatusProto.FATAL, RpcErrorCodeProto.ERROR_RPC_SERVER,
- null, t.getClass().getName(), StringUtils.stringifyException(t));
- call.sendResponse();
- }
-
/**
* Get service class for connection
* @return the serviceClass
@@ -2388,16 +2485,6 @@ public abstract class Server {
if (LOG.isDebugEnabled()) {
LOG.debug(Thread.currentThread().getName() + ": " + call + " for RpcKind " + call.rpcKind);
}
- if (!call.connection.channel.isOpen()) {
- LOG.info(Thread.currentThread().getName() + ": skipped " + call);
- continue;
- }
- String errorClass = null;
- String error = null;
- RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
- RpcErrorCodeProto detailedErr = null;
- Writable value = null;
-
CurCall.set(call);
if (call.traceScope != null) {
call.traceScope.reattach();
@@ -2406,53 +2493,11 @@ public abstract class Server {
}
// always update the current call context
CallerContext.setCurrent(call.callerContext);
-
- try {
- // Make the call as the user via Subject.doAs, thus associating
- // the call with the Subject
- if (call.connection.user == null) {
- value = call(call.rpcKind, call.connection.protocolName, call.rpcRequest,
- call.timestamp);
- } else {
- value =
- call.connection.user.doAs
- (new PrivilegedExceptionAction<Writable>() {
- @Override
- public Writable run() throws Exception {
- // make the call
- return call(call.rpcKind, call.connection.protocolName,
- call.rpcRequest, call.timestamp);
-
- }
- }
- );
- }
- } catch (Throwable e) {
- if (e instanceof UndeclaredThrowableException) {
- e = e.getCause();
- }
- logException(LOG, e, call);
- if (e instanceof RpcServerException) {
- RpcServerException rse = ((RpcServerException)e);
- returnStatus = rse.getRpcStatusProto();
- detailedErr = rse.getRpcErrorCodeProto();
- } else {
- returnStatus = RpcStatusProto.ERROR;
- detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
- }
- errorClass = e.getClass().getName();
- error = StringUtils.stringifyException(e);
- // Remove redundant error class name from the beginning of the stack trace
- String exceptionHdr = errorClass + ": ";
- if (error.startsWith(exceptionHdr)) {
- error = error.substring(exceptionHdr.length());
- }
- }
- CurCall.set(null);
- synchronized (call.connection.responseQueue) {
- setupResponse(call, returnStatus, detailedErr,
- value, errorClass, error);
- call.sendResponse();
+ UserGroupInformation remoteUser = call.getRemoteUser();
+ if (remoteUser != null) {
+ remoteUser.doAs(call);
+ } else {
+ call.run();
}
} catch (InterruptedException e) {
if (running) { // unexpected -- log it
@@ -2469,6 +2514,7 @@ public abstract class Server {
StringUtils.stringifyException(e));
}
} finally {
+ CurCall.set(null);
IOUtils.cleanup(LOG, traceScope);
}
}
@@ -2670,7 +2716,7 @@ public abstract class Server {
* @throws IOException
*/
private void setupResponse(
- Call call, RpcStatusProto status, RpcErrorCodeProto erCode,
+ RpcCall call, RpcStatusProto status, RpcErrorCodeProto erCode,
Writable rv, String errorClass, String error)
throws IOException {
RpcResponseHeaderProto.Builder headerBuilder =
@@ -2704,7 +2750,7 @@ public abstract class Server {
}
}
- private void setupResponse(Call call,
+ private void setupResponse(RpcCall call,
RpcResponseHeaderProto header, Writable rv) throws IOException {
ResponseBuffer buf = responseBuffer.get().reset();
try {
@@ -2738,7 +2784,7 @@ public abstract class Server {
* @throws IOException
*/
private void setupResponseOldVersionFatal(ByteArrayOutputStream response,
- Call call,
+ RpcCall call,
Writable rv, String errorClass, String error)
throws IOException {
final int OLD_VERSION_FATAL_STATUS = -1;
@@ -2751,7 +2797,7 @@ public abstract class Server {
call.setResponse(ByteBuffer.wrap(response.toByteArray()));
}
- private void wrapWithSasl(Call call) throws IOException {
+ private void wrapWithSasl(RpcCall call) throws IOException {
if (call.connection.saslServer != null) {
byte[] token = call.rpcResponse.array();
// synchronization may be needed since there can be multiple Handler
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/52] [abbrv] hadoop git commit: HADOOP-12726. Unsupported FS
operations should throw UnsupportedOperationException. Contributed by Daniel
Templeton.
Posted by aw...@apache.org.
HADOOP-12726. Unsupported FS operations should throw UnsupportedOperationException. Contributed by Daniel Templeton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c37346d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c37346d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c37346d0
Branch: refs/heads/HADOOP-13341
Commit: c37346d0e3f9d39d0aec7a9c5bda3e9772aa969b
Parents: 143c59e
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Aug 23 14:05:57 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Aug 23 14:12:52 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/ChecksumFileSystem.java | 6 ++++--
.../src/main/java/org/apache/hadoop/fs/ChecksumFs.java | 3 ++-
.../main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java | 3 ++-
.../main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java | 4 ++--
.../apache/hadoop/metrics2/sink/RollingFileSystemSink.java | 6 +++---
.../hadoop-common/src/site/markdown/filesystem/filesystem.md | 8 ++++----
.../main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 ++-
.../org/apache/hadoop/fs/s3native/NativeS3FileSystem.java | 3 ++-
8 files changed, 21 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index 1f14c4d..e4c0b33 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -356,12 +356,14 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
@Override
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by ChecksumFileSystem");
}
@Override
public boolean truncate(Path f, long newLength) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Truncate is not supported "
+ + "by ChecksumFileSystem");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
index 2b632a1..6e98db5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
@@ -297,7 +297,8 @@ public abstract class ChecksumFs extends FilterFs {
@Override
public boolean truncate(Path f, long newLength) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Truncate is not supported "
+ + "by ChecksumFs");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index d429c42..f1afacd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -288,7 +288,8 @@ public class FTPFileSystem extends FileSystem {
@Override
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by FTPFileSystem");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
index 8b6267a..30cf4d3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
@@ -77,7 +77,6 @@ public class SFTPFileSystem extends FileSystem {
"Can't make directory for path \"%s\" under \"%s\".";
public static final String E_DIR_NOTEMPTY = "Directory: %s is not empty.";
public static final String E_FILE_CHECK_FAILED = "File check failed";
- public static final String E_NOT_SUPPORTED = "Not supported";
public static final String E_SPATH_NOTEXIST = "Source path %s does not exist";
public static final String E_DPATH_EXIST =
"Destination path %s already exist, cannot rename!";
@@ -578,7 +577,8 @@ public class SFTPFileSystem extends FileSystem {
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress)
throws IOException {
- throw new IOException(E_NOT_SUPPORTED);
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by SFTPFileSystem");
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
index 9bfd519..91942f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
@@ -485,10 +485,10 @@ public class RollingFileSystemSink implements MetricsSink, Closeable {
try {
fs.append(basePath);
+ } catch (UnsupportedOperationException ex) {
+ canAppend = false;
} catch (IOException ex) {
- if (ex.getMessage().equals("Not supported")) {
- canAppend = false;
- }
+ // Ignore. The operation is supported.
}
return canAppend;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index 72dbc3c..1587842 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -611,7 +611,7 @@ this precondition fails.
### `FSDataOutputStream append(Path p, int bufferSize, Progressable progress)`
-Implementations MAY throw `UnsupportedOperationException`.
+Implementations without a compliant call SHOULD throw `UnsupportedOperationException`.
#### Preconditions
@@ -630,7 +630,7 @@ by appending data to the existing list.
### `FSDataInputStream open(Path f, int bufferSize)`
-Implementations MAY throw `UnsupportedOperationException`.
+Implementations without a compliant call SHOULD throw `UnsupportedOperationException`.
#### Preconditions
@@ -944,7 +944,7 @@ to the `DFSFileSystem` implementation is an ongoing matter for debate.
Joins multiple blocks together to create a single file. This
is a little-used operation currently implemented only by HDFS.
-Implementations MAY throw `UnsupportedOperationException`
+Implementations without a compliant call SHOULD throw `UnsupportedOperationException`.
#### Preconditions
@@ -989,7 +989,7 @@ from specification.
Truncate file `p` to the specified `newLength`.
-Implementations MAY throw `UnsupportedOperationException`.
+Implementations without a compliant call SHOULD throw `UnsupportedOperationException`.
#### Preconditions
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 7231349..f8c65da 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -633,7 +633,8 @@ public class S3AFileSystem extends FileSystem {
*/
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by S3AFileSystem");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c37346d0/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
index bebb09e..f741298 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
@@ -421,7 +421,8 @@ public class NativeS3FileSystem extends FileSystem {
@Override
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException {
- throw new IOException("Not supported");
+ throw new UnsupportedOperationException("Append is not supported "
+ + "by NativeS3FileSystem");
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/52] [abbrv] hadoop git commit: HADOOP-13533. Do not require user
to set HADOOP_SSH_OPTS to a non-null string,
allow setting of an empty string. (Albert Chu via aw) closes #121
Posted by aw...@apache.org.
HADOOP-13533. Do not require user to set HADOOP_SSH_OPTS to a non-null string, allow setting of an empty string. (Albert Chu via aw) closes #121
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52b25fe6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52b25fe6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52b25fe6
Branch: refs/heads/HADOOP-13341
Commit: 52b25fe6ae8f1fb720502032637419cf46fd0027
Parents: 601599b
Author: Albert Chu <ch...@llnl.gov>
Authored: Mon Aug 22 17:41:41 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 25 07:11:06 2016 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/bin/hadoop-functions.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b25fe6/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index b3aa018..75554f0 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -596,7 +596,7 @@ function hadoop_basic_init
HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}
HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_DAEMON_ROOT_LOGGER:-${HADOOP_LOGLEVEL},RFA}
HADOOP_SECURITY_LOGGER=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}
- HADOOP_SSH_OPTS=${HADOOP_SSH_OPTS:-"-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"}
+ HADOOP_SSH_OPTS=${HADOOP_SSH_OPTS-"-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"}
HADOOP_SECURE_LOG_DIR=${HADOOP_SECURE_LOG_DIR:-${HADOOP_LOG_DIR}}
HADOOP_SECURE_PID_DIR=${HADOOP_SECURE_PID_DIR:-${HADOOP_PID_DIR}}
HADOOP_SSH_PARALLEL=${HADOOP_SSH_PARALLEL:-10}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/52] [abbrv] hadoop git commit: YARN-5430. Return container's ip
and host from NM ContainerStatus call. Contributed by Jian He.
Posted by aw...@apache.org.
YARN-5430. Return container's ip and host from NM ContainerStatus call. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfb4d950
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfb4d950
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfb4d950
Branch: refs/heads/HADOOP-13341
Commit: bfb4d950596d6198e473045dac6eb36c685b0319
Parents: 27c3b86
Author: Varun Vasudev <vv...@apache.org>
Authored: Fri Aug 26 16:30:18 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Fri Aug 26 16:30:18 2016 +0530
----------------------------------------------------------------------
.../yarn/api/records/ContainerStatus.java | 26 +++++
.../src/main/proto/yarn_protos.proto | 1 +
.../records/impl/pb/ContainerStatusPBImpl.java | 107 ++++++++++++++++++-
.../protocolrecords/TestProtocolRecords.java | 17 +++
.../server/nodemanager/ContainerExecutor.java | 22 ++++
.../nodemanager/LinuxContainerExecutor.java | 5 +
.../containermanager/container/Container.java | 2 +
.../container/ContainerImpl.java | 14 ++-
.../privileged/PrivilegedOperationExecutor.java | 10 +-
.../runtime/DefaultLinuxContainerRuntime.java | 6 ++
.../DelegatingLinuxContainerRuntime.java | 6 ++
.../runtime/DockerLinuxContainerRuntime.java | 38 +++++++
.../runtime/docker/DockerInspectCommand.java | 10 ++
.../monitor/ContainersMonitorImpl.java | 23 ++--
.../runtime/ContainerRuntime.java | 6 ++
.../impl/container-executor.c | 13 ++-
.../docker/TestDockerInspectCommand.java | 9 ++
.../nodemanager/webapp/MockContainer.java | 6 ++
18 files changed, 304 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
index 4f56535..839fd04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.util.Records;
+import java.util.List;
+
/**
* {@code ContainerStatus} represents the current status of a
* {@code Container}.
@@ -151,4 +153,28 @@ public abstract class ContainerStatus {
@Private
@Unstable
public abstract void setCapability(Resource capability);
+
+ /**
+ * Get all the IP addresses with which the container run.
+ * @return The IP address where the container runs.
+ */
+ @Public
+ @Unstable
+ public abstract List<String> getIPs();
+
+ @Private
+ @Unstable
+ public abstract void setIPs(List<String> ips);
+
+ /**
+ * Get the hostname where the container runs.
+ * @return The hostname where the container runs.
+ */
+ @Public
+ @Unstable
+ public abstract String getHost();
+
+ @Private
+ @Unstable
+ public abstract void setHost(String host);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index d9ce11f..8a6ae3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -524,6 +524,7 @@ message ContainerStatusProto {
optional int32 exit_status = 4 [default = -1000];
optional ResourceProto capability = 5;
optional ExecutionTypeProto executionType = 6 [default = GUARANTEED];
+ repeated StringStringMapProto container_attributes = 7;
}
enum ContainerExitStatusProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
index f1fdae9..7ec6619 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
@@ -21,11 +21,13 @@ package org.apache.hadoop.yarn.api.records.impl.pb;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto;
@@ -33,7 +35,11 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProtoOrBuilder;
-import com.google.protobuf.TextFormat;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
@Private
@Unstable
@@ -43,8 +49,11 @@ public class ContainerStatusPBImpl extends ContainerStatus {
boolean viaProto = false;
private ContainerId containerId = null;
-
-
+ private static final String HOST = "HOST";
+ private static final String IPS = "IPS";
+ private Map<String, String> containerAttributes = new HashMap<>();
+
+
public ContainerStatusPBImpl() {
builder = ContainerStatusProto.newBuilder();
}
@@ -94,6 +103,9 @@ public class ContainerStatusPBImpl extends ContainerStatus {
if (containerId != null) {
builder.setContainerId(convertToProtoFormat(this.containerId));
}
+ if (containerAttributes != null && !containerAttributes.isEmpty()) {
+ addContainerAttributesToProto();
+ }
}
private synchronized void mergeLocalToProto() {
@@ -111,6 +123,57 @@ public class ContainerStatusPBImpl extends ContainerStatus {
viaProto = false;
}
+ private void addContainerAttributesToProto() {
+ maybeInitBuilder();
+ builder.clearContainerAttributes();
+ if (containerAttributes == null) {
+ return;
+ }
+ Iterable<YarnProtos.StringStringMapProto> iterable =
+ new Iterable<YarnProtos.StringStringMapProto>() {
+
+ @Override
+ public Iterator<YarnProtos.StringStringMapProto> iterator() {
+ return new Iterator<YarnProtos.StringStringMapProto>() {
+
+ private Iterator<String> keyIter =
+ containerAttributes.keySet().iterator();
+
+ @Override public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override public YarnProtos.StringStringMapProto next() {
+ String key = keyIter.next();
+ String value = containerAttributes.get(key);
+
+ if (value == null) {
+ value = "";
+ }
+
+ return YarnProtos.StringStringMapProto.newBuilder().setKey(key)
+ .setValue((value)).build();
+ }
+
+ @Override public boolean hasNext() {
+ return keyIter.hasNext();
+ }
+ };
+ }
+ };
+ builder.addAllContainerAttributes(iterable);
+ }
+
+ private void initContainerAttributes() {
+ ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
+ List<YarnProtos.StringStringMapProto> list = p.getContainerAttributesList();
+ for (YarnProtos.StringStringMapProto c : list) {
+ if (!containerAttributes.containsKey(c.getKey())) {
+ this.containerAttributes.put(c.getKey(), c.getValue());
+ }
+ }
+ }
+
@Override
public synchronized ExecutionType getExecutionType() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
@@ -211,6 +274,44 @@ public class ContainerStatusPBImpl extends ContainerStatus {
builder.setCapability(convertToProtoFormat(capability));
}
+ @Override
+ public synchronized List<String> getIPs() {
+ if (!containerAttributes.containsKey(IPS)) {
+ initContainerAttributes();
+ }
+ String ips = containerAttributes.get((IPS));
+ return ips == null ? null : Arrays.asList(ips.split(","));
+ }
+
+ @Override
+ public synchronized void setIPs(List<String> ips) {
+ maybeInitBuilder();
+ if (ips == null) {
+ containerAttributes.remove(IPS);
+ addContainerAttributesToProto();
+ return;
+ }
+ containerAttributes.put(IPS, StringUtils.join(",", ips));
+ }
+
+ @Override
+ public synchronized String getHost() {
+ if (containerAttributes.get(HOST) == null) {
+ initContainerAttributes();
+ }
+ return containerAttributes.get(HOST);
+ }
+
+ @Override
+ public synchronized void setHost(String host) {
+ maybeInitBuilder();
+ if (host == null) {
+ containerAttributes.remove(HOST);
+ return;
+ }
+ containerAttributes.put(HOST, host);
+ }
+
private ContainerStateProto convertToProtoFormat(ContainerState e) {
return ProtoUtils.convertToProtoFormat(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
index 27bdfff..35aa25e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
@@ -35,9 +35,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NMContainerStatusPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
@@ -161,4 +163,19 @@ public class TestProtocolRecords {
Assert.assertEquals(321,
pb.getNodeStatus().getQueuedContainersStatus().getWaitQueueLength());
}
+
+ @Test
+ public void testContainerStatus() {
+ ContainerStatus status = Records.newRecord(ContainerStatus.class);
+ List<String> ips = Arrays.asList("127.0.0.1", "139.5.25.2");
+ status.setIPs(ips);
+ status.setHost("locahost123");
+ ContainerStatusPBImpl pb =
+ new ContainerStatusPBImpl(((ContainerStatusPBImpl) status).getProto());
+ Assert.assertEquals(ips, pb.getIPs());
+ Assert.assertEquals("locahost123", pb.getHost());
+
+ status.setIPs(null);
+ Assert.assertNull(status.getIPs());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index f08db5a..c73c4c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -22,6 +22,8 @@ import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -626,6 +628,26 @@ public abstract class ContainerExecutor implements Configurable {
}
}
+ // LinuxContainerExecutor overrides this method and behaves differently.
+ public String[] getIpAndHost(Container container) {
+ return getLocalIpAndHost(container);
+ }
+
+ // ipAndHost[0] contains ip.
+ // ipAndHost[1] contains hostname.
+ public static String[] getLocalIpAndHost(Container container) {
+ String[] ipAndHost = new String[2];
+ try {
+ InetAddress address = InetAddress.getLocalHost();
+ ipAndHost[0] = address.getHostAddress();
+ ipAndHost[1] = address.getHostName();
+ } catch (UnknownHostException e) {
+ LOG.error("Unable to get Local hostname and ip for " + container
+ .getContainerId(), e);
+ }
+ return ipAndHost;
+ }
+
/**
* Mark the container as inactive. For inactive containers this
* method has no effect.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 2c27f6c..6c658bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -524,6 +524,11 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ public String[] getIpAndHost(Container container) {
+ return linuxContainerRuntime.getIpAndHost(container);
+ }
+
+ @Override
public int reacquireContainer(ContainerReacquisitionContext ctx)
throws IOException, InterruptedException {
ContainerId containerId = ctx.getContainerId();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 2278786..84d3cb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -68,6 +68,8 @@ public interface Container extends EventHandler<ContainerEvent> {
void setLogDir(String logDir);
+ void setIpAndHost(String[] ipAndHost);
+
String toString();
Priority getPriority();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 193dfea..3908971 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
@@ -115,6 +116,8 @@ public class ContainerImpl implements Container {
private int remainingRetryAttempts;
private String workDir;
private String logDir;
+ private String host;
+ private String ips;
/** The NM-wide configuration - not specific to this container */
private final Configuration daemonConf;
@@ -507,9 +510,12 @@ public class ContainerImpl implements Container {
public ContainerStatus cloneAndGetContainerStatus() {
this.readLock.lock();
try {
- return BuilderUtils.newContainerStatus(this.containerId,
+ ContainerStatus status = BuilderUtils.newContainerStatus(this.containerId,
getCurrentState(), diagnostics.toString(), exitCode, getResource(),
this.containerTokenIdentifier.getExecutionType());
+ status.setIPs(ips == null ? null : Arrays.asList(ips.split(",")));
+ status.setHost(host);
+ return status;
} finally {
this.readLock.unlock();
}
@@ -567,6 +573,12 @@ public class ContainerImpl implements Container {
}
@Override
+ public void setIpAndHost(String[] ipAndHost) {
+ this.ips = ipAndHost[0];
+ this.host = ipAndHost[1];
+ }
+
+ @Override
public String getLogDir() {
return logDir;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
index f865c14..1d874a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
@@ -157,13 +157,15 @@ public class PrivilegedOperationExecutor {
}
} catch (ExitCodeException e) {
if (operation.isFailureLoggingEnabled()) {
-
StringBuilder logBuilder = new StringBuilder("Shell execution returned "
+ "exit code: ")
.append(exec.getExitCode())
- .append(". Privileged Execution Operation Output: ")
- .append(System.lineSeparator()).append(exec.getOutput());
-
+ .append(". Privileged Execution Operation Stderr: ")
+ .append(System.lineSeparator())
+ .append(e.getMessage())
+ .append(System.lineSeparator())
+ .append("Stdout: " + exec.getOutput())
+ .append(System.lineSeparator());
logBuilder.append("Full command array for failed execution: ")
.append(System.lineSeparator());
logBuilder.append(Arrays.toString(fullCommandArray));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 57e3cc2..ee94aaa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
@@ -162,4 +163,9 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
throws ContainerExecutionException {
}
+
+ @Override
+ public String[] getIpAndHost(Container container) {
+ return ContainerExecutor.getLocalIpAndHost(container);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index faa7522..c12858e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -116,4 +116,10 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
runtime.reapContainer(ctx);
}
+
+ @Override
+ public String[] getIpAndHost(Container container) {
+ LinuxContainerRuntime runtime = pickContainerRuntime(container);
+ return runtime.getIpAndHost(container);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 2cce1f8..33b8add 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerClient;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerInspectCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStopCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
@@ -593,4 +594,41 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
public void reapContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
}
+
+
+ // ipAndHost[0] contains comma separated list of IPs
+ // ipAndHost[1] contains the hostname.
+ @Override
+ public String[] getIpAndHost(Container container) {
+ String containerId = container.getContainerId().toString();
+ DockerInspectCommand inspectCommand =
+ new DockerInspectCommand(containerId).getIpAndHost();
+ try {
+ String commandFile = dockerClient.writeCommandToTempFile(inspectCommand,
+ containerId);
+ PrivilegedOperation privOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.RUN_DOCKER_CMD);
+ privOp.appendArgs(commandFile);
+ String output = privilegedOperationExecutor
+ .executePrivilegedOperation(null, privOp, null,
+ container.getLaunchContext().getEnvironment(), true, false);
+ LOG.info("Docker inspect output for " + containerId + ": " + output);
+ int index = output.lastIndexOf(',');
+ if (index == -1) {
+ LOG.error("Incorrect format for ip and host");
+ return null;
+ }
+ String ips = output.substring(0, index).trim();
+ String host = output.substring(index+1).trim();
+ String[] ipAndHost = new String[2];
+ ipAndHost[0] = ips;
+ ipAndHost[1] = host;
+ return ipAndHost;
+ } catch (ContainerExecutionException e) {
+ LOG.error("Error when writing command to temp file", e);
+ } catch (PrivilegedOperationException e) {
+ LOG.error("Error when executing command.", e);
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
index 94b87b7..812a35f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerInspectCommand.java
@@ -38,4 +38,14 @@ public class DockerInspectCommand extends DockerCommand {
super.addCommandArguments(containerName);
return this;
}
+
+ public DockerInspectCommand getIpAndHost() {
+ // Be sure to not use space in the argument, otherwise the
+ // extract_values_delim method in container-executor binary
+ // cannot parse the arguments correctly.
+ super.addCommandArguments("--format='{{range(.NetworkSettings.Networks)}}"
+ + "{{.IPAddress}},{{end}}{{.Config.Hostname}}'");
+ super.addCommandArguments(containerName);
+ return this;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index e6a66bd..fb8cfc4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -18,10 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -45,8 +43,10 @@ import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
public class ContainersMonitorImpl extends AbstractService implements
ContainersMonitor {
@@ -447,6 +447,17 @@ public class ContainersMonitorImpl extends AbstractService implements
containerMetricsUnregisterDelayMs);
usageMetrics.recordProcessId(pId);
}
+ Container container = context.getContainers().get(containerId);
+ String[] ipAndHost = containerExecutor.getIpAndHost(container);
+ if (ipAndHost != null && ipAndHost[0] != null
+ && ipAndHost[1] != null) {
+ container.setIpAndHost(ipAndHost);
+ LOG.info(containerId + "'s ip = " + ipAndHost[0]
+ + ", and hostname = " + ipAndHost[1]);
+ } else {
+ LOG.info("Can not get both ip and hostname: " + Arrays
+ .toString(ipAndHost));
+ }
}
}
// End of initializing any uninitialized processTrees
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
index 653a4c2..b15690f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
/**
* An abstraction for various container runtime implementations. Examples
@@ -73,4 +74,9 @@ public interface ContainerRuntime {
*/
void reapContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException;
+
+ /**
+ * Return the host and ip of the container
+ */
+ String[] getIpAndHost(Container container);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 40b8edc0..f8c7f6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -419,6 +419,13 @@ int change_user(uid_t user, gid_t group) {
return 0;
}
+char* check_docker_binary(char *docker_binary) {
+ if (docker_binary == NULL) {
+ return "docker";
+ }
+ return docker_binary;
+}
+
/**
* Utility function to concatenate argB to argA using the concat_pattern.
*/
@@ -1123,6 +1130,8 @@ char* parse_docker_command_file(const char* command_file) {
int run_docker(const char *command_file) {
char* docker_command = parse_docker_command_file(command_file);
char* docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
+ docker_binary = check_docker_binary(docker_binary);
+
char* docker_command_with_binary = calloc(sizeof(char), EXECUTOR_PATH_MAX);
snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command);
char **args = extract_values_delim(docker_command_with_binary, " ");
@@ -1288,9 +1297,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
char *docker_command = parse_docker_command_file(command_file);
char *docker_binary = get_value(DOCKER_BINARY_KEY, &executor_cfg);
- if (docker_binary == NULL) {
- docker_binary = "docker";
- }
+ docker_binary = check_docker_binary(docker_binary);
fprintf(LOGFILE, "Creating script paths...\n");
exit_code = create_script_paths(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
index 3321364..619f202 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerInspectCommand.java
@@ -47,4 +47,13 @@ public class TestDockerInspectCommand {
assertEquals("inspect --format='{{.State.Status}}' foo",
dockerInspectCommand.getCommandWithArguments());
}
+
+ @Test
+ public void testGetIpAndHost() throws Exception {
+ dockerInspectCommand.getIpAndHost();
+ assertEquals(
+ "inspect --format='{{range(.NetworkSettings.Networks)}}{{.IPAddress}}"
+ + ",{{end}}{{.Config.Hostname}}' foo",
+ dockerInspectCommand.getCommandWithArguments());
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfb4d950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index b21ba4b..7513fdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -174,7 +174,13 @@ public class MockContainer implements Container {
public void setLogDir(String logDir) {
}
+ @Override
public Priority getPriority() {
return Priority.UNDEFINED;
}
+
+ @Override
+ public void setIpAndHost(String[] ipAndHost) {
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/52] [abbrv] hadoop git commit: YARN-5564. Fix typo in
RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE. Contributed by Ray
Chiang
Posted by aw...@apache.org.
YARN-5564. Fix typo in RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE. Contributed by Ray Chiang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27c3b862
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27c3b862
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27c3b862
Branch: refs/heads/HADOOP-13341
Commit: 27c3b86252386c9c064a6420b3c650644cbb9ef3
Parents: f4a21d3
Author: Naganarasimha <na...@apache.org>
Authored: Fri Aug 26 08:47:21 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Fri Aug 26 08:47:21 2016 +0530
----------------------------------------------------------------------
.../scheduler/fair/FairSchedulerConfiguration.java | 4 ++--
.../resourcemanager/scheduler/fair/FairSchedulerTestBase.java | 4 ++--
.../server/resourcemanager/scheduler/fair/TestFairScheduler.java | 2 +-
.../scheduler/fair/TestFairSchedulerPreemption.java | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27c3b862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 22cb10c..b18dd7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -52,7 +52,7 @@ public class FairSchedulerConfiguration extends Configuration {
* multiple of increment allocation. Only container sizes above this are
* allowed to reserve a node */
public static final String
- RM_SCHEDULER_RESERVATION_THRESHOLD_INCERMENT_MULTIPLE =
+ RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE =
YarnConfiguration.YARN_PREFIX +
"scheduler.reservation-threshold.increment-multiple";
public static final float
@@ -190,7 +190,7 @@ public class FairSchedulerConfiguration extends Configuration {
public float getReservationThresholdIncrementMultiple() {
return getFloat(
- RM_SCHEDULER_RESERVATION_THRESHOLD_INCERMENT_MULTIPLE,
+ RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE,
DEFAULT_RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27c3b862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index ec0e6aa..8e6272a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -84,7 +84,7 @@ public class FairSchedulerTestBase {
conf.setFloat(
FairSchedulerConfiguration
- .RM_SCHEDULER_RESERVATION_THRESHOLD_INCERMENT_MULTIPLE,
+ .RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE,
TEST_RESERVATION_THRESHOLD);
return conf;
}
@@ -280,4 +280,4 @@ public class FairSchedulerTestBase {
Assert.assertEquals(resource.getVirtualCores(),
app.getCurrentConsumption().getVirtualCores());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27c3b862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index da9d667..c348572 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -1455,7 +1455,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Set threshold to 2 * 1024 ==> 2048 MB & 2 * 1 ==> 2 vcores (test will
// use vcores)
conf.setFloat(FairSchedulerConfiguration.
- RM_SCHEDULER_RESERVATION_THRESHOLD_INCERMENT_MULTIPLE,
+ RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE,
2f);
scheduler.init(conf);
scheduler.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27c3b862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
index 79f4601..2cbe507 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
@@ -135,7 +135,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
conf.setFloat(
FairSchedulerConfiguration
- .RM_SCHEDULER_RESERVATION_THRESHOLD_INCERMENT_MULTIPLE,
+ .RM_SCHEDULER_RESERVATION_THRESHOLD_INCREMENT_MULTIPLE,
TEST_RESERVATION_THRESHOLD);
resourceManager = new MockRM(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/52] [abbrv] hadoop git commit: HADOOP-13532. Fix typo in
hadoop_connect_to_hosts error message (Albert Chu via aw) closes #120
Posted by aw...@apache.org.
HADOOP-13532. Fix typo in hadoop_connect_to_hosts error message (Albert Chu via aw) closes #120
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/601599b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/601599b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/601599b0
Branch: refs/heads/HADOOP-13341
Commit: 601599b03685c66444d94902485c388f47878cd6
Parents: 525d52b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Aug 25 06:58:42 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 25 06:58:42 2016 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/bin/hadoop-functions.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/601599b0/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index c380599..b3aa018 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -695,7 +695,7 @@ function hadoop_connect_to_hosts
#
# User can specify hostnames or a file where the hostnames are (not both)
if [[ -n "${HADOOP_WORKERS}" && -n "${HADOOP_WORKER_NAMES}" ]] ; then
- hadoop_error "ERROR: Both HADOOP_WORKERS and HADOOP_WORKER_NAME were defined. Aborting."
+ hadoop_error "ERROR: Both HADOOP_WORKERS and HADOOP_WORKER_NAMES were defined. Aborting."
exit 1
elif [[ -z "${HADOOP_WORKER_NAMES}" ]]; then
if [[ -n "${HADOOP_WORKERS}" ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/52] [abbrv] hadoop git commit: HDFS-8312. Added permission check
for moving file to Trash. (Weiwei Yang via Eric Yang)
Posted by aw...@apache.org.
HDFS-8312. Added permission check for moving file to Trash. (Weiwei Yang via Eric Yang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c49333be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c49333be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c49333be
Branch: refs/heads/HADOOP-13341
Commit: c49333becfa7652460976a61eb86522010bcfeed
Parents: 4070caa
Author: Eric Yang <ey...@apache.org>
Authored: Mon Aug 22 18:29:56 2016 -0700
Committer: Eric Yang <ey...@apache.org>
Committed: Mon Aug 22 18:29:56 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/Options.java | 3 +-
.../apache/hadoop/fs/TrashPolicyDefault.java | 10 ++-
.../ClientNamenodeProtocolTranslatorPB.java | 7 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 1 +
...tNamenodeProtocolServerSideTranslatorPB.java | 14 +++-
.../hdfs/server/namenode/FSDirRenameOp.java | 28 +++++--
.../apache/hadoop/hdfs/TestDFSPermission.java | 81 ++++++++++++++++++++
7 files changed, 132 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index da75d1c..dc50286 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -213,7 +213,8 @@ public final class Options {
*/
public static enum Rename {
NONE((byte) 0), // No options
- OVERWRITE((byte) 1); // Overwrite the rename destination
+ OVERWRITE((byte) 1), // Overwrite the rename destination
+ TO_TRASH ((byte) 2); // Rename to trash
private final byte code;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index 14f4c0c..66ef890 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -106,6 +106,7 @@ public class TrashPolicyDefault extends TrashPolicy {
return deletionInterval != 0;
}
+ @SuppressWarnings("deprecation")
@Override
public boolean moveToTrash(Path path) throws IOException {
if (!isEnabled())
@@ -156,10 +157,11 @@ public class TrashPolicyDefault extends TrashPolicy {
trashPath = new Path(orig + Time.now());
}
- if (fs.rename(path, trashPath)) { // move to current trash
- LOG.info("Moved: '" + path + "' to trash at: " + trashPath);
- return true;
- }
+ // move to current trash
+ fs.rename(path, trashPath,
+ Rename.TO_TRASH);
+ LOG.info("Moved: '" + path + "' to trash at: " + trashPath);
+ return true;
} catch (IOException e) {
cause = e;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index bcf5269..57f8fd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -523,16 +523,21 @@ public class ClientNamenodeProtocolTranslatorPB implements
public void rename2(String src, String dst, Rename... options)
throws IOException {
boolean overwrite = false;
+ boolean toTrash = false;
if (options != null) {
for (Rename option : options) {
if (option == Rename.OVERWRITE) {
overwrite = true;
+ } else if (option == Rename.TO_TRASH) {
+ toTrash = true;
}
}
}
Rename2RequestProto req = Rename2RequestProto.newBuilder().
setSrc(src).
- setDst(dst).setOverwriteDest(overwrite).
+ setDst(dst).
+ setOverwriteDest(overwrite).
+ setMoveToTrash(toTrash).
build();
try {
if (Client.isAsynchronousMode()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index aac48a4..83f5828 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -245,6 +245,7 @@ message Rename2RequestProto {
required string src = 1;
required string dst = 2;
required bool overwriteDest = 3;
+ optional bool moveToTrash = 4;
}
message Rename2ResponseProto { // void response
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 7a4ff82..641cd20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.protocolPB;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
@@ -602,10 +603,21 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
@Override
public Rename2ResponseProto rename2(RpcController controller,
Rename2RequestProto req) throws ServiceException {
+ // resolve rename options
+ ArrayList<Rename> optionList = new ArrayList<Rename>();
+ if(req.getOverwriteDest()) {
+ optionList.add(Rename.OVERWRITE);
+ } else if(req.hasMoveToTrash()) {
+ optionList.add(Rename.TO_TRASH);
+ }
+
+ if(optionList.isEmpty()) {
+ optionList.add(Rename.NONE);
+ }
try {
server.rename2(req.getSrc(), req.getDst(),
- req.getOverwriteDest() ? Rename.OVERWRITE : Rename.NONE);
+ optionList.toArray(new Rename[optionList.size()]));
} catch (IOException e) {
throw new ServiceException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index d90139b..f98f8b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -261,11 +261,29 @@ class FSDirRenameOp {
src = srcIIP.getPath();
dst = dstIIP.getPath();
if (fsd.isPermissionEnabled()) {
- // Rename does not operate on link targets
- // Do not resolveLink when checking permissions of src and dst
- // Check write access to parent of src
- fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
- false);
+ boolean renameToTrash = false;
+ if (null != options &&
+ Arrays.asList(options).
+ contains(Options.Rename.TO_TRASH)) {
+ renameToTrash = true;
+ }
+
+ if(renameToTrash) {
+ // if destination is the trash directory,
+ // besides the permission check on "rename"
+ // we need to enforce the check for "delete"
+ // otherwise, it would expose a
+ // security hole that stuff moved to trash
+ // will be deleted by superuser
+ fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null,
+ FsAction.ALL, true);
+ } else {
+ // Rename does not operate on link targets
+ // Do not resolveLink when checking permissions of src and dst
+ // Check write access to parent of src
+ fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null,
+ null, false);
+ }
// Check write access to ancestor of dst
fsd.checkPermission(pc, dstIIP, false, FsAction.WRITE, null, null, null,
false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c49333be/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
index e6524f3..d0d00e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.security.AccessControlException;
@@ -288,6 +289,86 @@ public class TestDFSPermission {
FsPermission.createImmutable((short)0777));
}
+ @Test(timeout=30000)
+ public void testTrashPermission() throws Exception {
+ // /BSS user1:group2 777
+ // /BSS/user1 user1:group2 755
+ // /BSS/user1/test user1:group1 600
+ Path rootDir = new Path("/BSS");
+ Path user1Dir = new Path("/BSS/user1");
+ Path user1File = new Path("/BSS/user1/test");
+
+ try {
+ conf.set(CommonConfigurationKeys.FS_TRASH_INTERVAL_KEY, "10");
+ fs = FileSystem.get(conf);
+
+ fs.mkdirs(rootDir);
+ fs.setPermission(rootDir, new FsPermission((short) 0777));
+
+ login(USER1);
+ fs.mkdirs(user1Dir);
+ fs.setPermission(user1Dir, new FsPermission((short) 0755));
+ fs.setOwner(user1Dir, USER1.getShortUserName(), GROUP2_NAME);
+
+ create(OpType.CREATE, user1File);
+ fs.setOwner(user1File, USER1.getShortUserName(), GROUP1_NAME);
+ fs.setPermission(user1File, new FsPermission((short) 0600));
+
+ try {
+ // login as user2, attempt to delete /BSS/user1
+ // this should fail because user2 has no permission to
+ // its sub directory.
+ login(USER2);
+ fs.delete(user1Dir, true);
+ fail("User2 should not be allowed to delete user1's dir.");
+ } catch (AccessControlException e) {
+ e.printStackTrace();
+ assertTrue("Permission denied messages must carry the username",
+ e.getMessage().contains(USER2_NAME));
+ }
+
+ // ensure the /BSS/user1 still exists
+ assertTrue(fs.exists(user1Dir));
+
+ try {
+ login(SUPERUSER);
+ Trash trash = new Trash(fs, conf);
+ Path trashRoot = trash.getCurrentTrashDir(user1Dir);
+ while(true) {
+ trashRoot = trashRoot.getParent();
+ if(trashRoot.getParent().isRoot()) {
+ break;
+ }
+ }
+ fs.mkdirs(trashRoot);
+ fs.setPermission(trashRoot, new FsPermission((short) 0777));
+
+ // login as user2, attempt to move /BSS/user1 to trash
+ // this should also fail otherwise the directory will be
+ // removed by trash emptier (emptier is running by superuser)
+ login(USER2);
+ Trash userTrash = new Trash(fs, conf);
+ assertTrue(userTrash.isEnabled());
+ userTrash.moveToTrash(user1Dir);
+ fail("User2 should not be allowed to move"
+ + "user1's dir to trash");
+ } catch (IOException e) {
+ // expect the exception is caused by permission denied
+ assertTrue(e.getCause() instanceof AccessControlException);
+ e.printStackTrace();
+ assertTrue("Permission denied messages must carry the username",
+ e.getCause().getMessage().contains(USER2_NAME));
+ }
+
+ // ensure /BSS/user1 still exists
+ assertEquals(fs.exists(user1Dir), true);
+ } finally {
+ login(SUPERUSER);
+ fs.delete(rootDir, true);
+ conf.set(CommonConfigurationKeys.FS_TRASH_INTERVAL_KEY, "0");
+ }
+ }
+
/* check if the ownership of a file/directory is set correctly */
@Test
public void testOwnership() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/52] [abbrv] hadoop git commit: HADOOP-13428. Fixed hadoop-common
build files to generate jdiff by working around a jdiff-bug. Contributed by
Wangda Tan.
Posted by aw...@apache.org.
HADOOP-13428. Fixed hadoop-common build files to generate jdiff by working around a jdiff-bug. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99603e90
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99603e90
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99603e90
Branch: refs/heads/HADOOP-13341
Commit: 99603e902244f17b04cfd55122f47355d070b588
Parents: 2da32a6
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Fri Aug 19 19:08:53 2016 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Fri Aug 19 19:08:53 2016 -0700
----------------------------------------------------------------------
.../dev-support/jdiff-workaround.patch | 98 +
.../jdiff/Apache_Hadoop_Common_2.7.2.xml | 46648 +++++++++++++++++
hadoop-common-project/hadoop-common/pom.xml | 2 +
hadoop-project-dist/pom.xml | 44 +-
4 files changed, 46789 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99603e90/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
new file mode 100644
index 0000000..8f87d40
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
@@ -0,0 +1,98 @@
+diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
+index a277abd..ed7c709 100644
+--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
+@@ -43,18 +43,6 @@
+ public abstract MetricsSystem init(String prefix);
+
+ /**
+- * Register a metrics source
+- * @param <T> the actual type of the source object
+- * @param source object to register
+- * @param name of the source. Must be unique or null (then extracted from
+- * the annotations of the source object.)
+- * @param desc the description of the source (or null. See above.)
+- * @return the source object
+- * @exception MetricsException
+- */
+- public abstract <T> T register(String name, String desc, T source);
+-
+- /**
+ * Unregister a metrics source
+ * @param name of the source. This is the name you use to call register()
+ */
+@@ -77,18 +65,19 @@
+ */
+ @InterfaceAudience.Private
+ public abstract MetricsSource getSource(String name);
++
+
+ /**
+- * Register a metrics sink
+- * @param <T> the type of the sink
+- * @param sink to register
+- * @param name of the sink. Must be unique.
+- * @param desc the description of the sink
+- * @return the sink
++ * Register a metrics source
++ * @param <T> the actual type of the source object
++ * @param source object to register
++ * @param name of the source. Must be unique or null (then extracted from
++ * the annotations of the source object.)
++ * @param desc the description of the source (or null. See above.)
++ * @return the source object
+ * @exception MetricsException
+ */
+- public abstract <T extends MetricsSink>
+- T register(String name, String desc, T sink);
++ public abstract <T> T register(String name, String desc, T source);
+
+ /**
+ * Register a callback interface for JMX events
+diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+index 6986edb..eeea81f 100644
+--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+@@ -270,27 +270,6 @@ void registerSource(String name, String desc, MetricsSource source) {
+ LOG.debug("Registered source "+ name);
+ }
+
+- @Override public synchronized <T extends MetricsSink>
+- T register(final String name, final String description, final T sink) {
+- LOG.debug(name +", "+ description);
+- if (allSinks.containsKey(name)) {
+- LOG.warn("Sink "+ name +" already exists!");
+- return sink;
+- }
+- allSinks.put(name, sink);
+- if (config != null) {
+- registerSink(name, description, sink);
+- }
+- // We want to re-register the sink to pick up new config
+- // when the metrics system restarts.
+- register(name, new AbstractCallback() {
+- @Override public void postStart() {
+- register(name, description, sink);
+- }
+- });
+- return sink;
+- }
+-
+ synchronized void registerSink(String name, String desc, MetricsSink sink) {
+ checkNotNull(config, "config");
+ MetricsConfig conf = sinkConfigs.get(name);
+diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
+index c19d238..f8412f1 100644
+--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
++++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
+@@ -130,8 +130,8 @@ public void testTagsForPrefix() throws Exception {
+ GangliaMetricsTestHelper.setDatagramSocket(gsink31, mockds31);
+
+ // register the sinks
+- ms.register("gsink30", "gsink30 desc", gsink30);
+- ms.register("gsink31", "gsink31 desc", gsink31);
++ //ms.register("gsink30", "gsink30 desc", gsink30);
++ //ms.register("gsink31", "gsink31 desc", gsink31);
+ ms.publishMetricsNow(); // publish the metrics
+
+ ms.stop();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/52] [abbrv] hadoop git commit: MAPREDUCE-6761. Regression when
handling providers - invalid configuration ServiceConfiguration causes
Cluster initialization failure. Contributed by Peter Vary
Posted by aw...@apache.org.
MAPREDUCE-6761. Regression when handling providers - invalid configuration ServiceConfiguration causes Cluster initialization failure. Contributed by Peter Vary
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fa9bf44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fa9bf44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fa9bf44
Branch: refs/heads/HADOOP-13341
Commit: 6fa9bf4407a0b09be8ce4e52d2fde89adba34f1a
Parents: 0ce1ab9
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Aug 24 14:39:28 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Aug 24 14:40:51 2016 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/mapreduce/Cluster.java | 18 ++++-
.../apache/hadoop/mapreduce/TestCluster.java | 81 ++++++++++++++++++++
2 files changed, 95 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa9bf44/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
index dcce2aa..3a2f982 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
@@ -24,8 +24,10 @@ import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
+import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -53,7 +55,7 @@ public class Cluster {
@InterfaceStability.Evolving
public static enum JobTrackerStatus {INITIALIZING, RUNNING};
-
+
private ClientProtocolProvider clientProtocolProvider;
private ClientProtocol client;
private UserGroupInformation ugi;
@@ -64,7 +66,8 @@ public class Cluster {
private Path jobHistoryDir = null;
private static final Log LOG = LogFactory.getLog(Cluster.class);
- private static ServiceLoader<ClientProtocolProvider> frameworkLoader =
+ @VisibleForTesting
+ static Iterable<ClientProtocolProvider> frameworkLoader =
ServiceLoader.load(ClientProtocolProvider.class);
private volatile List<ClientProtocolProvider> providerList = null;
@@ -74,8 +77,15 @@ public class Cluster {
if (providerList == null) {
List<ClientProtocolProvider> localProviderList =
new ArrayList<ClientProtocolProvider>();
- for (ClientProtocolProvider provider : frameworkLoader) {
- localProviderList.add(provider);
+ try {
+ for (ClientProtocolProvider provider : frameworkLoader) {
+ localProviderList.add(provider);
+ }
+ } catch(ServiceConfigurationError e) {
+ LOG.info("Failed to instantiate ClientProtocolProvider, please "
+ + "check the /META-INF/services/org.apache."
+ + "hadoop.mapreduce.protocol.ClientProtocolProvider "
+ + "files on the classpath", e);
}
providerList = localProviderList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fa9bf44/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCluster.java
new file mode 100644
index 0000000..3aa0ff1
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCluster.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce;
+
+import org.apache.hadoop.conf.Configuration;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.times;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
+import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.Iterator;
+import java.util.ServiceConfigurationError;
+
+/**
+ * Testing the Cluster initialization.
+ */
+public class TestCluster {
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testProtocolProviderCreation() throws Exception {
+ Iterator iterator = mock(Iterator.class);
+ when(iterator.hasNext()).thenReturn(true, true, true, true);
+ when(iterator.next()).thenReturn(getClientProtocolProvider())
+ .thenThrow(new ServiceConfigurationError("Test error"))
+ .thenReturn(getClientProtocolProvider());
+
+ Iterable frameworkLoader = mock(Iterable.class);
+ when(frameworkLoader.iterator()).thenReturn(iterator);
+
+ Cluster.frameworkLoader = frameworkLoader;
+ Cluster testCluster = new Cluster(new Configuration());
+
+ // Check that we get the acceptable client, even after
+ // failure in instantiation.
+ assertNotNull("ClientProtocol is expected", testCluster.getClient());
+ // Check if we do not try to load the providers after a failure.
+ verify(iterator, times(2)).next();
+ }
+
+ public ClientProtocolProvider getClientProtocolProvider() {
+ return new ClientProtocolProvider() {
+ @Override
+ public ClientProtocol create(Configuration conf) throws IOException {
+ return mock(ClientProtocol.class);
+ }
+
+ @Override
+ public ClientProtocol create(InetSocketAddress addr, Configuration
+ conf) throws IOException {
+ return mock(ClientProtocol.class);
+ }
+
+ @Override
+ public void close(ClientProtocol clientProtocol) throws IOException {
+ }
+ };
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/52] [abbrv] hadoop git commit: Revert "HADOOP-13465. Design
Server.Call to be extensible for unified call queue. Contributed by Daryn
Sharp."
Posted by aw...@apache.org.
Revert "HADOOP-13465. Design Server.Call to be extensible for unified call queue. Contributed by Daryn Sharp."
This reverts commit d288a0ba8364d81aacda9f4a21022eecb6dc4e22.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81485dbf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81485dbf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81485dbf
Branch: refs/heads/HADOOP-13341
Commit: 81485dbfc1ffb8daa609be8eb31094cc28646dd3
Parents: 1360bd2
Author: Kihwal Lee <ki...@apache.org>
Authored: Thu Aug 25 16:04:54 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Thu Aug 25 16:04:54 2016 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Server.java | 336 ++++++++-----------
1 file changed, 145 insertions(+), 191 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81485dbf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 09fe889..4c73f6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -354,9 +354,10 @@ public abstract class Server {
*/
public static InetAddress getRemoteIp() {
Call call = CurCall.get();
- return (call != null ) ? call.getHostInetAddress() : null;
+ return (call != null && call.connection != null) ? call.connection
+ .getHostInetAddress() : null;
}
-
+
/**
* Returns the clientId from the current RPC request
*/
@@ -379,9 +380,10 @@ public abstract class Server {
*/
public static UserGroupInformation getRemoteUser() {
Call call = CurCall.get();
- return (call != null) ? call.getRemoteUser() : null;
+ return (call != null && call.connection != null) ? call.connection.user
+ : null;
}
-
+
/** Return true if the invocation was through an RPC.
*/
public static boolean isRpcInvocation() {
@@ -481,7 +483,7 @@ public abstract class Server {
if ((rpcMetrics.getProcessingSampleCount() > minSampleSize) &&
(processingTime > threeSigma)) {
if(LOG.isWarnEnabled()) {
- String client = CurCall.get().toString();
+ String client = CurCall.get().connection.toString();
LOG.warn(
"Slow RPC : " + methodName + " took " + processingTime +
" milliseconds to process from client " + client);
@@ -655,65 +657,62 @@ public abstract class Server {
CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT);
}
- /** A generic call queued for handling. */
- public static class Call implements Schedulable,
- PrivilegedExceptionAction<Void> {
- final int callId; // the client's call id
- final int retryCount; // the retry count of the call
- long timestamp; // time received when response is null
- // time served when response is not null
+ /** A call queued for handling. */
+ public static class Call implements Schedulable {
+ private final int callId; // the client's call id
+ private final int retryCount; // the retry count of the call
+ private final Writable rpcRequest; // Serialized Rpc request from client
+ private final Connection connection; // connection to client
+ private long timestamp; // time received when response is null
+ // time served when response is not null
+ private ByteBuffer rpcResponse; // the response for this call
private AtomicInteger responseWaitCount = new AtomicInteger(1);
- final RPC.RpcKind rpcKind;
- final byte[] clientId;
+ private final RPC.RpcKind rpcKind;
+ private final byte[] clientId;
private final TraceScope traceScope; // the HTrace scope on the server side
private final CallerContext callerContext; // the call context
private int priorityLevel;
// the priority level assigned by scheduler, 0 by default
- Call(Call call) {
- this(call.callId, call.retryCount, call.rpcKind, call.clientId,
- call.traceScope, call.callerContext);
+ private Call(Call call) {
+ this(call.callId, call.retryCount, call.rpcRequest, call.connection,
+ call.rpcKind, call.clientId, call.traceScope, call.callerContext);
}
- Call(int id, int retryCount, RPC.RpcKind kind, byte[] clientId) {
- this(id, retryCount, kind, clientId, null, null);
+ public Call(int id, int retryCount, Writable param,
+ Connection connection) {
+ this(id, retryCount, param, connection, RPC.RpcKind.RPC_BUILTIN,
+ RpcConstants.DUMMY_CLIENT_ID);
}
- @VisibleForTesting // primarily TestNamenodeRetryCache
- public Call(int id, int retryCount, Void ignore1, Void ignore2,
+ public Call(int id, int retryCount, Writable param, Connection connection,
RPC.RpcKind kind, byte[] clientId) {
- this(id, retryCount, kind, clientId, null, null);
+ this(id, retryCount, param, connection, kind, clientId, null, null);
}
- Call(int id, int retryCount, RPC.RpcKind kind, byte[] clientId,
- TraceScope traceScope, CallerContext callerContext) {
+ public Call(int id, int retryCount, Writable param, Connection connection,
+ RPC.RpcKind kind, byte[] clientId, TraceScope traceScope,
+ CallerContext callerContext) {
this.callId = id;
this.retryCount = retryCount;
+ this.rpcRequest = param;
+ this.connection = connection;
this.timestamp = Time.now();
+ this.rpcResponse = null;
this.rpcKind = kind;
this.clientId = clientId;
this.traceScope = traceScope;
this.callerContext = callerContext;
}
-
+
@Override
public String toString() {
- return "Call#" + callId + " Retry#" + retryCount;
+ return rpcRequest + " from " + connection + " Call#" + callId + " Retry#"
+ + retryCount;
}
- public Void run() throws Exception {
- return null;
- }
- // should eventually be abstract but need to avoid breaking tests
- public UserGroupInformation getRemoteUser() {
- return null;
- }
- public InetAddress getHostInetAddress() {
- return null;
- }
- public String getHostAddress() {
- InetAddress addr = getHostInetAddress();
- return (addr != null) ? addr.getHostAddress() : null;
+ public void setResponse(ByteBuffer response) {
+ this.rpcResponse = response;
}
/**
@@ -725,36 +724,34 @@ public abstract class Server {
*/
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public final void postponeResponse() {
+ public void postponeResponse() {
int count = responseWaitCount.incrementAndGet();
assert count > 0 : "response has already been sent";
}
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public final void sendResponse() throws IOException {
+ public void sendResponse() throws IOException {
int count = responseWaitCount.decrementAndGet();
assert count >= 0 : "response has already been sent";
if (count == 0) {
- doResponse(null);
+ connection.sendResponse(this);
}
}
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public final void abortResponse(Throwable t) throws IOException {
+ public void abortResponse(Throwable t) throws IOException {
// don't send response if the call was already sent or aborted.
if (responseWaitCount.getAndSet(-1) > 0) {
- doResponse(t);
+ connection.abortResponse(this, t);
}
}
- void doResponse(Throwable t) throws IOException {}
-
// For Schedulable
@Override
public UserGroupInformation getUserGroupInformation() {
- return getRemoteUser();
+ return connection.user;
}
@Override
@@ -767,114 +764,6 @@ public abstract class Server {
}
}
- /** A RPC extended call queued for handling. */
- private class RpcCall extends Call {
- final Connection connection; // connection to client
- final Writable rpcRequest; // Serialized Rpc request from client
- ByteBuffer rpcResponse; // the response for this call
-
- RpcCall(RpcCall call) {
- super(call);
- this.connection = call.connection;
- this.rpcRequest = call.rpcRequest;
- }
-
- RpcCall(Connection connection, int id) {
- this(connection, id, RpcConstants.INVALID_RETRY_COUNT);
- }
-
- RpcCall(Connection connection, int id, int retryCount) {
- this(connection, id, retryCount, null,
- RPC.RpcKind.RPC_BUILTIN, RpcConstants.DUMMY_CLIENT_ID,
- null, null);
- }
-
- RpcCall(Connection connection, int id, int retryCount,
- Writable param, RPC.RpcKind kind, byte[] clientId,
- TraceScope traceScope, CallerContext context) {
- super(id, retryCount, kind, clientId, traceScope, context);
- this.connection = connection;
- this.rpcRequest = param;
- }
-
- @Override
- public UserGroupInformation getRemoteUser() {
- return connection.user;
- }
-
- @Override
- public InetAddress getHostInetAddress() {
- return connection.getHostInetAddress();
- }
-
- @Override
- public Void run() throws Exception {
- if (!connection.channel.isOpen()) {
- Server.LOG.info(Thread.currentThread().getName() + ": skipped " + this);
- return null;
- }
- String errorClass = null;
- String error = null;
- RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
- RpcErrorCodeProto detailedErr = null;
- Writable value = null;
-
- try {
- value = call(
- rpcKind, connection.protocolName, rpcRequest, timestamp);
- } catch (Throwable e) {
- if (e instanceof UndeclaredThrowableException) {
- e = e.getCause();
- }
- logException(Server.LOG, e, this);
- if (e instanceof RpcServerException) {
- RpcServerException rse = ((RpcServerException)e);
- returnStatus = rse.getRpcStatusProto();
- detailedErr = rse.getRpcErrorCodeProto();
- } else {
- returnStatus = RpcStatusProto.ERROR;
- detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
- }
- errorClass = e.getClass().getName();
- error = StringUtils.stringifyException(e);
- // Remove redundant error class name from the beginning of the
- // stack trace
- String exceptionHdr = errorClass + ": ";
- if (error.startsWith(exceptionHdr)) {
- error = error.substring(exceptionHdr.length());
- }
- }
- setupResponse(this, returnStatus, detailedErr,
- value, errorClass, error);
- sendResponse();
- return null;
- }
-
- void setResponse(ByteBuffer response) throws IOException {
- this.rpcResponse = response;
- }
-
- @Override
- void doResponse(Throwable t) throws IOException {
- RpcCall call = this;
- if (t != null) {
- // clone the call to prevent a race with another thread stomping
- // on the response while being sent. the original call is
- // effectively discarded since the wait count won't hit zero
- call = new RpcCall(this);
- setupResponse(call,
- RpcStatusProto.FATAL, RpcErrorCodeProto.ERROR_RPC_SERVER,
- null, t.getClass().getName(), StringUtils.stringifyException(t));
- }
- connection.sendResponse(this);
- }
-
- @Override
- public String toString() {
- return super.toString() + " " + rpcRequest + " from " + connection;
- }
- }
-
/** Listens on the socket. Creates jobs for the handler threads*/
private class Listener extends Thread {
@@ -1205,22 +1094,22 @@ public abstract class Server {
if(LOG.isDebugEnabled()) {
LOG.debug("Checking for old call responses.");
}
- ArrayList<RpcCall> calls;
+ ArrayList<Call> calls;
// get the list of channels from list of keys.
synchronized (writeSelector.keys()) {
- calls = new ArrayList<RpcCall>(writeSelector.keys().size());
+ calls = new ArrayList<Call>(writeSelector.keys().size());
iter = writeSelector.keys().iterator();
while (iter.hasNext()) {
SelectionKey key = iter.next();
- RpcCall call = (RpcCall)key.attachment();
+ Call call = (Call)key.attachment();
if (call != null && key.channel() == call.connection.channel) {
calls.add(call);
}
}
}
-
- for (RpcCall call : calls) {
+
+ for(Call call : calls) {
doPurge(call, now);
}
} catch (OutOfMemoryError e) {
@@ -1238,7 +1127,7 @@ public abstract class Server {
}
private void doAsyncWrite(SelectionKey key) throws IOException {
- RpcCall call = (RpcCall)key.attachment();
+ Call call = (Call)key.attachment();
if (call == null) {
return;
}
@@ -1266,10 +1155,10 @@ public abstract class Server {
// Remove calls that have been pending in the responseQueue
// for a long time.
//
- private void doPurge(RpcCall call, long now) {
- LinkedList<RpcCall> responseQueue = call.connection.responseQueue;
+ private void doPurge(Call call, long now) {
+ LinkedList<Call> responseQueue = call.connection.responseQueue;
synchronized (responseQueue) {
- Iterator<RpcCall> iter = responseQueue.listIterator(0);
+ Iterator<Call> iter = responseQueue.listIterator(0);
while (iter.hasNext()) {
call = iter.next();
if (now > call.timestamp + PURGE_INTERVAL) {
@@ -1283,12 +1172,12 @@ public abstract class Server {
// Processes one response. Returns true if there are no more pending
// data for this channel.
//
- private boolean processResponse(LinkedList<RpcCall> responseQueue,
+ private boolean processResponse(LinkedList<Call> responseQueue,
boolean inHandler) throws IOException {
boolean error = true;
boolean done = false; // there is more data for this channel.
int numElements = 0;
- RpcCall call = null;
+ Call call = null;
try {
synchronized (responseQueue) {
//
@@ -1371,7 +1260,7 @@ public abstract class Server {
//
// Enqueue a response from the application.
//
- void doRespond(RpcCall call) throws IOException {
+ void doRespond(Call call) throws IOException {
synchronized (call.connection.responseQueue) {
// must only wrap before adding to the responseQueue to prevent
// postponed responses from being encrypted and sent out of order.
@@ -1469,7 +1358,7 @@ public abstract class Server {
private SocketChannel channel;
private ByteBuffer data;
private ByteBuffer dataLengthBuffer;
- private LinkedList<RpcCall> responseQueue;
+ private LinkedList<Call> responseQueue;
// number of outstanding rpcs
private AtomicInteger rpcCount = new AtomicInteger();
private long lastContact;
@@ -1496,8 +1385,8 @@ public abstract class Server {
public UserGroupInformation attemptingUser = null; // user name before auth
// Fake 'call' for failed authorization response
- private final RpcCall authFailedCall =
- new RpcCall(this, AUTHORIZATION_FAILED_CALL_ID);
+ private final Call authFailedCall = new Call(AUTHORIZATION_FAILED_CALL_ID,
+ RpcConstants.INVALID_RETRY_COUNT, null, this);
private boolean sentNegotiate = false;
private boolean useWrap = false;
@@ -1520,7 +1409,7 @@ public abstract class Server {
this.hostAddress = addr.getHostAddress();
}
this.remotePort = socket.getPort();
- this.responseQueue = new LinkedList<RpcCall>();
+ this.responseQueue = new LinkedList<Call>();
if (socketSendBufferSize != 0) {
try {
socket.setSendBufferSize(socketSendBufferSize);
@@ -1815,7 +1704,8 @@ public abstract class Server {
}
private void doSaslReply(Message message) throws IOException {
- final RpcCall saslCall = new RpcCall(this, AuthProtocol.SASL.callId);
+ final Call saslCall = new Call(AuthProtocol.SASL.callId,
+ RpcConstants.INVALID_RETRY_COUNT, null, this);
setupResponse(saslCall,
RpcStatusProto.SUCCESS, null,
RpcWritable.wrap(message), null, null);
@@ -2032,20 +1922,23 @@ public abstract class Server {
if (clientVersion >= 9) {
// Versions >>9 understand the normal response
- RpcCall fakeCall = new RpcCall(this, -1);
+ Call fakeCall = new Call(-1, RpcConstants.INVALID_RETRY_COUNT, null,
+ this);
setupResponse(fakeCall,
RpcStatusProto.FATAL, RpcErrorCodeProto.FATAL_VERSION_MISMATCH,
null, VersionMismatch.class.getName(), errMsg);
fakeCall.sendResponse();
} else if (clientVersion >= 3) {
- RpcCall fakeCall = new RpcCall(this, -1);
+ Call fakeCall = new Call(-1, RpcConstants.INVALID_RETRY_COUNT, null,
+ this);
// Versions 3 to 8 use older response
setupResponseOldVersionFatal(buffer, fakeCall,
null, VersionMismatch.class.getName(), errMsg);
fakeCall.sendResponse();
} else if (clientVersion == 2) { // Hadoop 0.18.3
- RpcCall fakeCall = new RpcCall(this, 0);
+ Call fakeCall = new Call(0, RpcConstants.INVALID_RETRY_COUNT, null,
+ this);
DataOutputStream out = new DataOutputStream(buffer);
out.writeInt(0); // call ID
out.writeBoolean(true); // error
@@ -2057,7 +1950,7 @@ public abstract class Server {
}
private void setupHttpRequestOnIpcPortResponse() throws IOException {
- RpcCall fakeCall = new RpcCall(this, 0);
+ Call fakeCall = new Call(0, RpcConstants.INVALID_RETRY_COUNT, null, this);
fakeCall.setResponse(ByteBuffer.wrap(
RECEIVED_HTTP_REQ_RESPONSE.getBytes(StandardCharsets.UTF_8)));
fakeCall.sendResponse();
@@ -2205,7 +2098,7 @@ public abstract class Server {
}
} catch (WrappedRpcServerException wrse) { // inform client of error
Throwable ioe = wrse.getCause();
- final RpcCall call = new RpcCall(this, callId, retry);
+ final Call call = new Call(callId, retry, null, this);
setupResponse(call,
RpcStatusProto.FATAL, wrse.getRpcErrorCodeProto(), null,
ioe.getClass().getName(), ioe.getMessage());
@@ -2305,9 +2198,8 @@ public abstract class Server {
.build();
}
- RpcCall call = new RpcCall(this, header.getCallId(),
- header.getRetryCount(), rpcRequest,
- ProtoUtil.convert(header.getRpcKind()),
+ Call call = new Call(header.getCallId(), header.getRetryCount(),
+ rpcRequest, this, ProtoUtil.convert(header.getRpcKind()),
header.getClientId().toByteArray(), traceScope, callerContext);
// Save the priority level assignment by the scheduler
@@ -2431,10 +2323,21 @@ public abstract class Server {
}
}
- private void sendResponse(RpcCall call) throws IOException {
+ private void sendResponse(Call call) throws IOException {
responder.doRespond(call);
}
+ private void abortResponse(Call call, Throwable t) throws IOException {
+ // clone the call to prevent a race with the other thread stomping
+ // on the response while being sent. the original call is
+ // effectively discarded since the wait count won't hit zero
+ call = new Call(call);
+ setupResponse(call,
+ RpcStatusProto.FATAL, RpcErrorCodeProto.ERROR_RPC_SERVER,
+ null, t.getClass().getName(), StringUtils.stringifyException(t));
+ call.sendResponse();
+ }
+
/**
* Get service class for connection
* @return the serviceClass
@@ -2485,6 +2388,16 @@ public abstract class Server {
if (LOG.isDebugEnabled()) {
LOG.debug(Thread.currentThread().getName() + ": " + call + " for RpcKind " + call.rpcKind);
}
+ if (!call.connection.channel.isOpen()) {
+ LOG.info(Thread.currentThread().getName() + ": skipped " + call);
+ continue;
+ }
+ String errorClass = null;
+ String error = null;
+ RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
+ RpcErrorCodeProto detailedErr = null;
+ Writable value = null;
+
CurCall.set(call);
if (call.traceScope != null) {
call.traceScope.reattach();
@@ -2493,11 +2406,53 @@ public abstract class Server {
}
// always update the current call context
CallerContext.setCurrent(call.callerContext);
- UserGroupInformation remoteUser = call.getRemoteUser();
- if (remoteUser != null) {
- remoteUser.doAs(call);
- } else {
- call.run();
+
+ try {
+ // Make the call as the user via Subject.doAs, thus associating
+ // the call with the Subject
+ if (call.connection.user == null) {
+ value = call(call.rpcKind, call.connection.protocolName, call.rpcRequest,
+ call.timestamp);
+ } else {
+ value =
+ call.connection.user.doAs
+ (new PrivilegedExceptionAction<Writable>() {
+ @Override
+ public Writable run() throws Exception {
+ // make the call
+ return call(call.rpcKind, call.connection.protocolName,
+ call.rpcRequest, call.timestamp);
+
+ }
+ }
+ );
+ }
+ } catch (Throwable e) {
+ if (e instanceof UndeclaredThrowableException) {
+ e = e.getCause();
+ }
+ logException(LOG, e, call);
+ if (e instanceof RpcServerException) {
+ RpcServerException rse = ((RpcServerException)e);
+ returnStatus = rse.getRpcStatusProto();
+ detailedErr = rse.getRpcErrorCodeProto();
+ } else {
+ returnStatus = RpcStatusProto.ERROR;
+ detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
+ }
+ errorClass = e.getClass().getName();
+ error = StringUtils.stringifyException(e);
+ // Remove redundant error class name from the beginning of the stack trace
+ String exceptionHdr = errorClass + ": ";
+ if (error.startsWith(exceptionHdr)) {
+ error = error.substring(exceptionHdr.length());
+ }
+ }
+ CurCall.set(null);
+ synchronized (call.connection.responseQueue) {
+ setupResponse(call, returnStatus, detailedErr,
+ value, errorClass, error);
+ call.sendResponse();
}
} catch (InterruptedException e) {
if (running) { // unexpected -- log it
@@ -2514,7 +2469,6 @@ public abstract class Server {
StringUtils.stringifyException(e));
}
} finally {
- CurCall.set(null);
IOUtils.cleanup(LOG, traceScope);
}
}
@@ -2716,7 +2670,7 @@ public abstract class Server {
* @throws IOException
*/
private void setupResponse(
- RpcCall call, RpcStatusProto status, RpcErrorCodeProto erCode,
+ Call call, RpcStatusProto status, RpcErrorCodeProto erCode,
Writable rv, String errorClass, String error)
throws IOException {
RpcResponseHeaderProto.Builder headerBuilder =
@@ -2750,7 +2704,7 @@ public abstract class Server {
}
}
- private void setupResponse(RpcCall call,
+ private void setupResponse(Call call,
RpcResponseHeaderProto header, Writable rv) throws IOException {
ResponseBuffer buf = responseBuffer.get().reset();
try {
@@ -2784,7 +2738,7 @@ public abstract class Server {
* @throws IOException
*/
private void setupResponseOldVersionFatal(ByteArrayOutputStream response,
- RpcCall call,
+ Call call,
Writable rv, String errorClass, String error)
throws IOException {
final int OLD_VERSION_FATAL_STATUS = -1;
@@ -2797,7 +2751,7 @@ public abstract class Server {
call.setResponse(ByteBuffer.wrap(response.toByteArray()));
}
- private void wrapWithSasl(RpcCall call) throws IOException {
+ private void wrapWithSasl(Call call) throws IOException {
if (call.connection.saslServer != null) {
byte[] token = call.rpcResponse.array();
// synchronization may be needed since there can be multiple Handler
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/52] [abbrv] hadoop git commit: HDFS-10762. Pass IIP for file
status related methods
Posted by aw...@apache.org.
HDFS-10762. Pass IIP for file status related methods
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec252ce0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec252ce0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec252ce0
Branch: refs/heads/HADOOP-13341
Commit: ec252ce0fc0998ce13f31af3440c08a236328e5a
Parents: 092b4d5
Author: Daryn Sharp <da...@yahoo-inc.com>
Authored: Wed Aug 24 08:46:47 2016 -0500
Committer: Daryn Sharp <da...@yahoo-inc.com>
Committed: Wed Aug 24 08:46:47 2016 -0500
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirAppendOp.java | 6 +--
.../server/namenode/FSDirStatAndListingOp.java | 56 +++++++++-----------
.../hdfs/server/namenode/FSDirWriteFileOp.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 14 ++---
.../hdfs/server/namenode/INodesInPath.java | 42 ++++++++++++---
.../hadoop/hdfs/TestReservedRawPaths.java | 21 ++++++++
6 files changed, 91 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec252ce0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 3a5d7dc..5192352 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -85,9 +85,10 @@ final class FSDirAppendOp {
final LocatedBlock lb;
final FSDirectory fsd = fsn.getFSDirectory();
final String src;
+ final INodesInPath iip;
fsd.writeLock();
try {
- final INodesInPath iip = fsd.resolvePathForWrite(pc, srcArg);
+ iip = fsd.resolvePathForWrite(pc, srcArg);
src = iip.getPath();
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
@@ -148,8 +149,7 @@ final class FSDirAppendOp {
fsd.writeUnlock();
}
- HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, src, false,
- FSDirectory.isReservedRawName(srcArg));
+ HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, iip);
if (lb != null) {
NameNode.stateChangeLog.debug(
"DIR* NameSystem.appendFile: file {} for {} at {} block {} block"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec252ce0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index c9eedf5..8a9393e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -108,16 +108,16 @@ class FSDirStatAndListingOp {
if (!DFSUtil.isValidName(src)) {
throw new InvalidPathException("Invalid file name: " + src);
}
+ final INodesInPath iip;
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- final INodesInPath iip = fsd.resolvePath(pc, srcArg, resolveLink);
- src = iip.getPath();
+ iip = fsd.resolvePath(pc, srcArg, resolveLink);
fsd.checkPermission(pc, iip, false, null, null, null, null, false);
} else {
src = FSDirectory.resolvePath(srcArg, fsd);
+ iip = fsd.getINodesInPath(src, resolveLink);
}
- return getFileInfo(fsd, src, FSDirectory.isReservedRawName(srcArg),
- resolveLink);
+ return getFileInfo(fsd, iip);
}
/**
@@ -230,7 +230,6 @@ class FSDirStatAndListingOp {
String src, byte[] startAfter, boolean needLocation, boolean isSuperUser)
throws IOException {
String srcs = FSDirectory.normalizePath(src);
- final boolean isRawPath = FSDirectory.isReservedRawName(src);
if (FSDirectory.isExactReservedName(srcs)) {
return getReservedListing(fsd);
}
@@ -257,7 +256,7 @@ class FSDirStatAndListingOp {
return new DirectoryListing(
new HdfsFileStatus[]{ createFileStatus(
fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- needLocation, parentStoragePolicy, snapshot, isRawPath, iip)
+ needLocation, parentStoragePolicy, iip)
}, 0);
}
@@ -282,7 +281,7 @@ class FSDirStatAndListingOp {
cur.getLocalNameBytes());
listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), nodeAttrs,
needLocation, getStoragePolicyID(curPolicy, parentStoragePolicy),
- snapshot, isRawPath, iipWithChild);
+ iipWithChild);
listingCnt++;
if (needLocation) {
// Once we hit lsLimit locations, stop.
@@ -339,7 +338,6 @@ class FSDirStatAndListingOp {
listing[i] = createFileStatus(
fsd, sRoot.getLocalNameBytes(), nodeAttrs,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
- Snapshot.CURRENT_STATE_ID, false,
INodesInPath.fromINode(sRoot));
}
return new DirectoryListing(
@@ -363,10 +361,8 @@ class FSDirStatAndListingOp {
* @return object containing information regarding the file
* or null if file not found
*/
- static HdfsFileStatus getFileInfo(
- FSDirectory fsd, String path, INodesInPath iip, boolean isRawPath,
- boolean includeStoragePolicy)
- throws IOException {
+ static HdfsFileStatus getFileInfo(FSDirectory fsd,
+ INodesInPath iip, boolean includeStoragePolicy) throws IOException {
fsd.readLock();
try {
final INode node = iip.getLastINode();
@@ -377,23 +373,21 @@ class FSDirStatAndListingOp {
byte policyId = includeStoragePolicy && !node.isSymlink() ?
node.getStoragePolicyID() :
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
- INodeAttributes nodeAttrs = getINodeAttributes(fsd, path,
+ INodeAttributes nodeAttrs = getINodeAttributes(fsd, iip.getPath(),
HdfsFileStatus.EMPTY_NAME,
node, iip.getPathSnapshotId());
return createFileStatus(fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- policyId, iip.getPathSnapshotId(), isRawPath, iip);
+ policyId, iip);
} finally {
fsd.readUnlock();
}
}
- static HdfsFileStatus getFileInfo(
- FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath)
+ static HdfsFileStatus getFileInfo(FSDirectory fsd, INodesInPath iip)
throws IOException {
fsd.readLock();
try {
HdfsFileStatus status = null;
- final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
status = FSDirectory.DOT_RESERVED_STATUS;
} else if (iip.isDotSnapshotDir()) {
@@ -401,7 +395,7 @@ class FSDirStatAndListingOp {
status = FSDirectory.DOT_SNAPSHOT_DIR_STATUS;
}
} else {
- status = getFileInfo(fsd, src, iip, isRawPath, true);
+ status = getFileInfo(fsd, iip, true);
}
return status;
} finally {
@@ -423,15 +417,12 @@ class FSDirStatAndListingOp {
*/
private static HdfsFileStatus createFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath,
- INodesInPath iip)
+ boolean needLocation, byte storagePolicy, INodesInPath iip)
throws IOException {
if (needLocation) {
- return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy,
- snapshot, isRawPath, iip);
+ return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
} else {
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
- snapshot, isRawPath, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
}
}
@@ -445,8 +436,7 @@ class FSDirStatAndListingOp {
INodesInPath iip) throws IOException {
INodeAttributes nodeAttrs = getINodeAttributes(
fsd, fullPath, path, iip.getLastINode(), snapshot);
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
- snapshot, isRawPath, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
}
/**
@@ -454,14 +444,15 @@ class FSDirStatAndListingOp {
* @param iip the INodesInPath containing the target INode and its ancestors
*/
static HdfsFileStatus createFileStatus(
- FSDirectory fsd, byte[] path,
- INodeAttributes nodeAttrs, byte storagePolicy, int snapshot,
- boolean isRawPath, INodesInPath iip) throws IOException {
+ FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
+ byte storagePolicy, INodesInPath iip) throws IOException {
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
final boolean isEncrypted;
final INode node = iip.getLastINode();
+ final int snapshot = iip.getPathSnapshotId();
+ final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
@@ -511,10 +502,9 @@ class FSDirStatAndListingOp {
* Create FileStatus with location info by file INode
* @param iip the INodesInPath containing the target INode and its ancestors
*/
- private static HdfsLocatedFileStatus createLocatedFileStatus(
+ private static HdfsFileStatus createLocatedFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- byte storagePolicy, int snapshot,
- boolean isRawPath, INodesInPath iip) throws IOException {
+ byte storagePolicy, INodesInPath iip) throws IOException {
assert fsd.hasReadLock();
long size = 0; // length is zero for directories
short replication = 0;
@@ -522,6 +512,8 @@ class FSDirStatAndListingOp {
LocatedBlocks loc = null;
final boolean isEncrypted;
final INode node = iip.getLastINode();
+ final int snapshot = iip.getPathSnapshotId();
+ final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec252ce0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 8429024..030d8cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -340,7 +340,6 @@ class FSDirWriteFileOp {
version = ezInfo.protocolVersion;
}
- boolean isRawPath = FSDirectory.isReservedRawName(src);
FSDirectory fsd = fsn.getFSDirectory();
INodesInPath iip = fsd.resolvePathForWrite(pc, src);
src = iip.getPath();
@@ -444,7 +443,7 @@ class FSDirWriteFileOp {
NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
src + " inode " + newNode.getId() + " " + holder);
}
- return FSDirStatAndListingOp.getFileInfo(fsd, src, false, isRawPath);
+ return FSDirStatAndListingOp.getFileInfo(fsd, iip);
}
static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec252ce0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index b6652e4..5dc5a03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -531,21 +531,24 @@ public class FSDirectory implements Closeable {
* @throws FileNotFoundException
* @throws AccessControlException
*/
- INodesInPath resolvePath(FSPermissionChecker pc, String src)
+ @VisibleForTesting
+ public INodesInPath resolvePath(FSPermissionChecker pc, String src)
throws UnresolvedLinkException, FileNotFoundException,
AccessControlException {
return resolvePath(pc, src, true);
}
- INodesInPath resolvePath(FSPermissionChecker pc, String src,
+ @VisibleForTesting
+ public INodesInPath resolvePath(FSPermissionChecker pc, String src,
boolean resolveLink) throws UnresolvedLinkException,
FileNotFoundException, AccessControlException {
byte[][] components = INode.getPathComponents(src);
- if (isPermissionEnabled && pc != null && isReservedRawName(components)) {
+ boolean isRaw = isReservedRawName(components);
+ if (isPermissionEnabled && pc != null && isRaw) {
pc.checkSuperuserPrivilege();
}
components = resolveComponents(components, this);
- return INodesInPath.resolve(rootDir, components, resolveLink);
+ return INodesInPath.resolve(rootDir, components, isRaw, resolveLink);
}
INodesInPath resolvePathForWrite(FSPermissionChecker pc, String src)
@@ -1662,8 +1665,7 @@ public class FSDirectory implements Closeable {
HdfsFileStatus getAuditFileInfo(INodesInPath iip)
throws IOException {
return (namesystem.isAuditEnabled() && namesystem.isExternalInvocation())
- ? FSDirStatAndListingOp.getFileInfo(this, iip.getPath(), iip, false,
- false) : null;
+ ? FSDirStatAndListingOp.getFileInfo(this, iip, false) : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec252ce0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index 86cab28..af8998f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -126,6 +126,12 @@ public class INodesInPath {
static INodesInPath resolve(final INodeDirectory startingDir,
final byte[][] components, final boolean resolveLink)
throws UnresolvedLinkException {
+ return resolve(startingDir, components, false, resolveLink);
+ }
+
+ static INodesInPath resolve(final INodeDirectory startingDir,
+ final byte[][] components, final boolean isRaw,
+ final boolean resolveLink) throws UnresolvedLinkException {
Preconditions.checkArgument(startingDir.compareTo(components[0]) == 0);
INode curNode = startingDir;
@@ -225,7 +231,7 @@ public class INodesInPath {
System.arraycopy(inodes, 0, newNodes, 0, newNodes.length);
inodes = newNodes;
}
- return new INodesInPath(inodes, components, isSnapshot, snapshotId);
+ return new INodesInPath(inodes, components, isRaw, isSnapshot, snapshotId);
}
private static boolean shouldUpdateLatestId(int sid, int snapshotId) {
@@ -249,7 +255,8 @@ public class INodesInPath {
INode[] inodes = new INode[iip.inodes.length];
System.arraycopy(iip.inodes, 0, inodes, 0, inodes.length);
inodes[pos] = inode;
- return new INodesInPath(inodes, iip.path, iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, iip.path, iip.isRaw,
+ iip.isSnapshot, iip.snapshotId);
}
/**
@@ -267,7 +274,8 @@ public class INodesInPath {
byte[][] path = new byte[iip.path.length + 1][];
System.arraycopy(iip.path, 0, path, 0, path.length - 1);
path[path.length - 1] = childName;
- return new INodesInPath(inodes, path, iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, path, iip.isRaw,
+ iip.isSnapshot, iip.snapshotId);
}
private final byte[][] path;
@@ -279,6 +287,13 @@ public class INodesInPath {
* true if this path corresponds to a snapshot
*/
private final boolean isSnapshot;
+
+ /**
+ * true if this is a /.reserved/raw path. path component resolution strips
+ * it from the path so need to track it separately.
+ */
+ private final boolean isRaw;
+
/**
* For snapshot paths, it is the id of the snapshot; or
* {@link Snapshot#CURRENT_STATE_ID} if the snapshot does not exist. For
@@ -287,17 +302,18 @@ public class INodesInPath {
*/
private final int snapshotId;
- private INodesInPath(INode[] inodes, byte[][] path, boolean isSnapshot,
- int snapshotId) {
+ private INodesInPath(INode[] inodes, byte[][] path, boolean isRaw,
+ boolean isSnapshot,int snapshotId) {
Preconditions.checkArgument(inodes != null && path != null);
this.inodes = inodes;
this.path = path;
+ this.isRaw = isRaw;
this.isSnapshot = isSnapshot;
this.snapshotId = snapshotId;
}
private INodesInPath(INode[] inodes, byte[][] path) {
- this(inodes, path, false, CURRENT_STATE_ID);
+ this(inodes, path, false, false, CURRENT_STATE_ID);
}
/**
@@ -400,7 +416,7 @@ public class INodesInPath {
final byte[][] apath = new byte[length][];
System.arraycopy(this.inodes, 0, anodes, 0, length);
System.arraycopy(this.path, 0, apath, 0, length);
- return new INodesInPath(anodes, apath, false, snapshotId);
+ return new INodesInPath(anodes, apath, isRaw, false, snapshotId);
}
/**
@@ -428,7 +444,7 @@ public class INodesInPath {
byte[][] existingPath = new byte[i][];
System.arraycopy(inodes, 0, existing, 0, i);
System.arraycopy(path, 0, existingPath, 0, i);
- return new INodesInPath(existing, existingPath, false, snapshotId);
+ return new INodesInPath(existing, existingPath, isRaw, false, snapshotId);
}
/**
@@ -438,10 +454,20 @@ public class INodesInPath {
return this.isSnapshot;
}
+ /**
+ * @return if .snapshot is the last path component.
+ */
boolean isDotSnapshotDir() {
return isDotSnapshotDir(getLastLocalName());
}
+ /**
+ * @return if this is a /.reserved/raw path.
+ */
+ public boolean isRaw() {
+ return isRaw;
+ }
+
private static String toString(INode inode) {
return inode == null? null: inode.getLocalName();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec252ce0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
index 449f715..c09d346 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
+import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
@@ -49,6 +50,8 @@ import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesNotEqual;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.apache.hadoop.test.GenericTestUtils.assertMatches;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestReservedRawPaths {
@@ -100,6 +103,24 @@ public class TestReservedRawPaths {
}
/**
+ * Verify resolving path will return an iip that tracks if the original
+ * path was a raw path.
+ */
+ @Test(timeout = 120000)
+ public void testINodesInPath() throws IOException {
+ FSDirectory fsd = cluster.getNamesystem().getFSDirectory();
+ final String path = "/path";
+
+ INodesInPath iip = fsd.resolvePath(null, path);
+ assertFalse(iip.isRaw());
+ assertEquals(path, iip.getPath());
+
+ iip = fsd.resolvePath(null, "/.reserved/raw" + path);
+ assertTrue(iip.isRaw());
+ assertEquals(path, iip.getPath());
+ }
+
+ /**
* Basic read/write tests of raw files.
* Create a non-encrypted file
* Create an encryption zone
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/52] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by aw...@apache.org.
HADOOP-13446. Support running isolated unit tests separate from AWS integration tests. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f9c346e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f9c346e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f9c346e
Branch: refs/heads/HADOOP-13341
Commit: 6f9c346e577325ec2059d83d5636b5ff7fa6cdce
Parents: f0efea4
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Aug 23 07:18:49 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Aug 23 07:18:49 2016 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 5 +
hadoop-tools/hadoop-aws/pom.xml | 95 +++-
.../src/site/markdown/tools/hadoop-aws/index.md | 67 ++-
.../fs/contract/s3a/ITestS3AContractCreate.java | 35 ++
.../fs/contract/s3a/ITestS3AContractDelete.java | 34 ++
.../fs/contract/s3a/ITestS3AContractDistCp.java | 46 ++
.../s3a/ITestS3AContractGetFileStatus.java | 51 ++
.../fs/contract/s3a/ITestS3AContractMkdir.java | 34 ++
.../fs/contract/s3a/ITestS3AContractOpen.java | 34 ++
.../fs/contract/s3a/ITestS3AContractRename.java | 62 +++
.../contract/s3a/ITestS3AContractRootDir.java | 72 +++
.../fs/contract/s3a/ITestS3AContractSeek.java | 34 ++
.../fs/contract/s3a/TestS3AContractCreate.java | 33 --
.../fs/contract/s3a/TestS3AContractDelete.java | 31 --
.../fs/contract/s3a/TestS3AContractDistCp.java | 46 --
.../s3a/TestS3AContractGetFileStatus.java | 47 --
.../fs/contract/s3a/TestS3AContractMkdir.java | 34 --
.../fs/contract/s3a/TestS3AContractOpen.java | 31 --
.../fs/contract/s3a/TestS3AContractRename.java | 61 ---
.../fs/contract/s3a/TestS3AContractRootDir.java | 72 ---
.../fs/contract/s3a/TestS3AContractSeek.java | 31 --
.../fs/contract/s3n/ITestS3NContractCreate.java | 41 ++
.../fs/contract/s3n/ITestS3NContractDelete.java | 34 ++
.../fs/contract/s3n/ITestS3NContractMkdir.java | 34 ++
.../fs/contract/s3n/ITestS3NContractOpen.java | 34 ++
.../fs/contract/s3n/ITestS3NContractRename.java | 35 ++
.../contract/s3n/ITestS3NContractRootDir.java | 35 ++
.../fs/contract/s3n/ITestS3NContractSeek.java | 34 ++
.../fs/contract/s3n/TestS3NContractCreate.java | 38 --
.../fs/contract/s3n/TestS3NContractDelete.java | 31 --
.../fs/contract/s3n/TestS3NContractMkdir.java | 34 --
.../fs/contract/s3n/TestS3NContractOpen.java | 31 --
.../fs/contract/s3n/TestS3NContractRename.java | 32 --
.../fs/contract/s3n/TestS3NContractRootDir.java | 35 --
.../fs/contract/s3n/TestS3NContractSeek.java | 31 --
.../ITestBlockingThreadPoolExecutorService.java | 182 +++++++
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 250 +++++++++
.../fs/s3a/ITestS3ABlockingThreadPool.java | 80 +++
.../apache/hadoop/fs/s3a/ITestS3ABlocksize.java | 96 ++++
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 435 +++++++++++++++
.../hadoop/fs/s3a/ITestS3ACredentialsInURL.java | 155 ++++++
.../hadoop/fs/s3a/ITestS3AEncryption.java | 104 ++++
.../ITestS3AEncryptionAlgorithmPropagation.java | 83 +++
.../s3a/ITestS3AEncryptionFastOutputStream.java | 35 ++
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 192 +++++++
.../hadoop/fs/s3a/ITestS3AFastOutputStream.java | 74 +++
.../fs/s3a/ITestS3AFileOperationCost.java | 191 +++++++
.../fs/s3a/ITestS3AFileSystemContract.java | 106 ++++
.../fs/s3a/ITestS3ATemporaryCredentials.java | 148 +++++
.../TestBlockingThreadPoolExecutorService.java | 182 -------
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 250 ---------
.../fs/s3a/TestS3ABlockingThreadPool.java | 80 ---
.../apache/hadoop/fs/s3a/TestS3ABlocksize.java | 93 ----
.../hadoop/fs/s3a/TestS3AConfiguration.java | 429 ---------------
.../hadoop/fs/s3a/TestS3ACredentialsInURL.java | 155 ------
.../apache/hadoop/fs/s3a/TestS3AEncryption.java | 104 ----
.../TestS3AEncryptionAlgorithmPropagation.java | 82 ---
.../s3a/TestS3AEncryptionFastOutputStream.java | 35 --
.../hadoop/fs/s3a/TestS3AFailureHandling.java | 194 -------
.../hadoop/fs/s3a/TestS3AFastOutputStream.java | 74 ---
.../hadoop/fs/s3a/TestS3AFileOperationCost.java | 191 -------
.../fs/s3a/TestS3AFileSystemContract.java | 104 ----
.../fs/s3a/TestS3ATemporaryCredentials.java | 148 -----
.../fs/s3a/fileContext/ITestS3AFileContext.java | 23 +
.../ITestS3AFileContextCreateMkdir.java | 35 ++
.../ITestS3AFileContextMainOperations.java | 60 +++
.../ITestS3AFileContextStatistics.java | 61 +++
.../s3a/fileContext/ITestS3AFileContextURI.java | 44 ++
.../fileContext/ITestS3AFileContextUtil.java | 34 ++
.../fs/s3a/fileContext/TestS3AFileContext.java | 23 -
.../TestS3AFileContextCreateMkdir.java | 35 --
.../TestS3AFileContextMainOperations.java | 60 ---
.../TestS3AFileContextStatistics.java | 60 ---
.../s3a/fileContext/TestS3AFileContextURI.java | 43 --
.../s3a/fileContext/TestS3AFileContextUtil.java | 34 --
.../s3a/scale/ITestS3ADeleteFilesOneByOne.java | 44 ++
.../fs/s3a/scale/ITestS3ADeleteManyFiles.java | 135 +++++
.../s3a/scale/ITestS3ADirectoryPerformance.java | 187 +++++++
.../scale/ITestS3AInputStreamPerformance.java | 534 +++++++++++++++++++
.../fs/s3a/scale/TestS3ADeleteManyFiles.java | 135 -----
.../s3a/scale/TestS3ADirectoryPerformance.java | 187 -------
.../scale/TestS3AInputStreamPerformance.java | 534 -------------------
.../org/apache/hadoop/fs/s3a/yarn/ITestS3A.java | 88 +++
.../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 148 +++++
.../org/apache/hadoop/fs/s3a/yarn/TestS3A.java | 85 ---
.../fs/s3a/yarn/TestS3AMiniYarnCluster.java | 144 -----
...ITestInMemoryNativeS3FileSystemContract.java | 33 ++
.../ITestJets3tNativeFileSystemStore.java | 133 +++++
.../ITestJets3tNativeS3FileSystemContract.java | 33 ++
.../TestInMemoryNativeS3FileSystemContract.java | 30 --
.../TestJets3tNativeFileSystemStore.java | 129 -----
.../TestJets3tNativeS3FileSystemContract.java | 30 --
92 files changed, 4503 insertions(+), 4304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c3146ca..65e9672 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1193,6 +1193,11 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${maven-failsafe-plugin.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-install-plugin</artifactId>
<version>${maven-install-plugin.version}</version>
</plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 8147d37..82a8581 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -46,7 +46,7 @@
</file>
</activation>
<properties>
- <maven.test.skip>true</maven.test.skip>
+ <skipITs>true</skipITs>
</properties>
</profile>
<profile>
@@ -57,11 +57,16 @@
</file>
</activation>
<properties>
- <maven.test.skip>false</maven.test.skip>
+ <skipITs>false</skipITs>
</properties>
</profile>
<profile>
<id>parallel-tests</id>
+ <activation>
+ <property>
+ <name>parallel-tests</name>
+ </property>
+ </activation>
<build>
<plugins>
<plugin>
@@ -96,12 +101,32 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkCount>${testsThreadCount}</forkCount>
+ <reuseForks>false</reuseForks>
+ <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
+ <systemPropertyVariables>
+ <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
+ <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
+ <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
+
+ <!-- Due to a Maven quirk, setting this to just -->
+ <!-- surefire.forkNumber won't do the parameter -->
+ <!-- substitution. Putting a prefix in front of it like -->
+ <!-- "fork-" makes it work. -->
+ <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
- <id>default-test</id>
- <phase>test</phase>
+ <id>default-integration-test</id>
<goals>
- <goal>test</goal>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
</goals>
<configuration>
<forkCount>${testsThreadCount}</forkCount>
@@ -129,32 +154,35 @@
<!-- Exclude all of these tests from parallel execution, -->
<!-- and instead run them sequentially in a separate -->
<!-- Surefire execution step later. -->
+ <includes>
+ <include>**/ITest*.java</include>
+ </includes>
<excludes>
- <exclude>**/TestJets3tNativeS3FileSystemContract.java</exclude>
- <exclude>**/TestS3ABlockingThreadPool.java</exclude>
- <exclude>**/TestS3AFastOutputStream.java</exclude>
- <exclude>**/TestS3AFileSystemContract.java</exclude>
- <exclude>**/TestS3AMiniYarnCluster.java</exclude>
- <exclude>**/Test*Root*.java</exclude>
+ <exclude>**/ITestJets3tNativeS3FileSystemContract.java</exclude>
+ <exclude>**/ITestS3ABlockingThreadPool.java</exclude>
+ <exclude>**/ITestS3AFastOutputStream.java</exclude>
+ <exclude>**/ITestS3AFileSystemContract.java</exclude>
+ <exclude>**/ITestS3AMiniYarnCluster.java</exclude>
+ <exclude>**/ITest*Root*.java</exclude>
</excludes>
</configuration>
</execution>
<execution>
- <id>sequential-tests</id>
- <phase>test</phase>
+ <id>sequential-integration-tests</id>
<goals>
- <goal>test</goal>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
</goals>
<configuration>
<!-- Do a sequential run for tests that cannot handle -->
<!-- parallel execution. -->
<includes>
- <include>**/TestJets3tNativeS3FileSystemContract.java</include>
- <include>**/TestS3ABlockingThreadPool.java</include>
- <include>**/TestS3AFastOutputStream.java</include>
- <include>**/TestS3AFileSystemContract.java</include>
- <include>**/TestS3AMiniYarnCluster.java</include>
- <include>**/Test*Root*.java</include>
+ <include>**/ITestJets3tNativeS3FileSystemContract.java</include>
+ <include>**/ITestS3ABlockingThreadPool.java</include>
+ <include>**/ITestS3AFastOutputStream.java</include>
+ <include>**/ITestS3AFileSystemContract.java</include>
+ <include>**/ITestS3AMiniYarnCluster.java</include>
+ <include>**/ITest*Root*.java</include>
</includes>
</configuration>
</execution>
@@ -163,6 +191,33 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>sequential-tests</id>
+ <activation>
+ <property>
+ <name>!parallel-tests</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 78c9f12..5100030 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -1319,11 +1319,18 @@ works with S3 to something lower. See [AWS documentation](http://docs.aws.amazon
## Testing the S3 filesystem clients
-Due to eventual consistency, tests may fail without reason. Transient
-failures, which no longer occur upon rerunning the test, should thus be ignored.
+This module includes both unit tests, which can run in isolation without
+connecting to the S3 service, and integration tests, which require a working
+connection to S3 to interact with a bucket. Unit test suites follow the naming
+convention `Test*.java`. Integration tests follow the naming convention
+`ITest*.java`.
-To test the S3* filesystem clients, you need to provide two files
-which pass in authentication details to the test runner
+Due to eventual consistency, integration tests may fail without reason.
+Transient failures, which no longer occur upon rerunning the test, should thus
+be ignored.
+
+To integration test the S3* filesystem clients, you need to provide two files
+which pass in authentication details to the test runner.
1. `auth-keys.xml`
1. `core-site.xml`
@@ -1343,7 +1350,8 @@ need to apply a specific, non-default property change during the tests.
The presence of this file triggers the testing of the S3 classes.
-Without this file, *none of the tests in this module will be executed*
+Without this file, *none of the integration tests in this module will be
+executed*.
The XML file must contain all the ID/key information needed to connect
each of the filesystem clients to the object stores, and a URL for
@@ -1496,23 +1504,50 @@ source code tree, it is not going to get accidentally committed.
After completing the configuration, execute the test run through Maven.
- mvn clean test
+ mvn clean verify
-It's also possible to execute multiple test suites in parallel by enabling the
-`parallel-tests` Maven profile. The tests spend most of their time blocked on
-network I/O with the S3 service, so running in parallel tends to complete full
-test runs faster.
+It's also possible to execute multiple test suites in parallel by passing the
+`parallel-tests` property on the command line. The tests spend most of their
+time blocked on network I/O with the S3 service, so running in parallel tends to
+complete full test runs faster.
- mvn -Pparallel-tests clean test
+ mvn -Dparallel-tests clean verify
Some tests must run with exclusive access to the S3 bucket, so even with the
-`parallel-tests` profile enabled, several test suites will run in serial in a
-separate Maven execution step after the parallel tests.
+`parallel-tests` property, several test suites will run in serial in a separate
+Maven execution step after the parallel tests.
+
+By default, `parallel-tests` runs 4 test suites concurrently. This can be tuned
+by passing the `testsThreadCount` property.
+
+ mvn -Dparallel-tests -DtestsThreadCount=8 clean verify
+
+To run just unit tests, which do not require S3 connectivity or AWS credentials,
+use any of the above invocations, but switch the goal to `test` instead of
+`verify`.
+
+ mvn clean test
+
+ mvn -Dparallel-tests clean test
+
+ mvn -Dparallel-tests -DtestsThreadCount=8 clean test
+
+To run only a specific named subset of tests, pass the `test` property for unit
+tests or the `it.test` property for integration tests.
+
+ mvn clean test -Dtest=TestS3AInputPolicies
+
+ mvn clean verify -Dit.test=ITestS3AFileContextStatistics
-By default, the `parallel-tests` profile runs 4 test suites concurrently. This
-can be tuned by passing the `testsThreadCount` argument.
+ mvn clean verify -Dtest=TestS3A* -Dit.test=ITestS3A*
- mvn -Pparallel-tests -DtestsThreadCount=8 clean test
+Note that when running a specific subset of tests, the patterns passed in `test`
+and `it.test` override the configuration of which tests need to run in isolation
+in a separate serial phase (mentioned above). This can cause unpredictable
+results, so the recommendation is to avoid passing `parallel-tests` in
+combination with `test` or `it.test`. If you know that you are specifying only
+tests that can run safely in parallel, then it will work. For wide patterns,
+like `ITestS3A*` shown above, it may cause unpredictable test failures.
### Testing against different regions
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
new file mode 100644
index 0000000..d2a858f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests creating files.
+ */
+public class ITestS3AContractCreate extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java
new file mode 100644
index 0000000..a47dcae
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDelete.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests covering deletes.
+ */
+public class ITestS3AContractDelete extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
new file mode 100644
index 0000000..28278fe
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import static org.apache.hadoop.fs.s3a.Constants.MIN_MULTIPART_THRESHOLD;
+import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_SIZE;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+
+/**
+ * Contract test suite covering S3A integration with DistCp.
+ */
+public class ITestS3AContractDistCp extends AbstractContractDistCpTest {
+
+ private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ newConf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_SETTING);
+ newConf.setLong(MULTIPART_SIZE, MULTIPART_SETTING);
+ return newConf;
+ }
+
+ @Override
+ protected S3AContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
new file mode 100644
index 0000000..c7ed5a3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+
+/**
+ * S3A contract tests covering getFileStatus.
+ */
+public class ITestS3AContractGetFileStatus
+ extends AbstractContractGetFileStatusTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ getLog().info("FS details {}", getFileSystem());
+ super.teardown();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ // aggressively low page size forces tests to go multipage
+ conf.setInt(Constants.MAX_PAGING_KEYS, 2);
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
new file mode 100644
index 0000000..d953e7e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test dir operations on S3A.
+ */
+public class ITestS3AContractMkdir extends AbstractContractMkdirTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java
new file mode 100644
index 0000000..a7bdc0d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractOpen.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests opening files.
+ */
+public class ITestS3AContractOpen extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java
new file mode 100644
index 0000000..5dba03d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRename.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
+
+/**
+ * S3A contract tests covering rename.
+ */
+public class ITestS3AContractRename extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Override
+ public void testRenameDirIntoExistingDir() throws Throwable {
+ describe("Verify renaming a dir into an existing dir puts the files"
+ +" from the source dir into the existing dir"
+ +" and leaves existing files alone");
+ FileSystem fs = getFileSystem();
+ String sourceSubdir = "source";
+ Path srcDir = path(sourceSubdir);
+ Path srcFilePath = new Path(srcDir, "source-256.txt");
+ byte[] srcDataset = dataset(256, 'a', 'z');
+ writeDataset(fs, srcFilePath, srcDataset, srcDataset.length, 1024, false);
+ Path destDir = path("dest");
+
+ Path destFilePath = new Path(destDir, "dest-512.txt");
+ byte[] destDateset = dataset(512, 'A', 'Z');
+ writeDataset(fs, destFilePath, destDateset, destDateset.length, 1024,
+ false);
+ assertIsFile(destFilePath);
+
+ boolean rename = fs.rename(srcDir, destDir);
+ assertFalse("s3a doesn't support rename to non-empty directory", rename);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java
new file mode 100644
index 0000000..8383a77
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractRootDir.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * root dir operations against an S3 bucket.
+ */
+public class ITestS3AContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AContractRootDir.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Override
+ public void testListEmptyRootDirectory() throws IOException {
+ for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
+ try {
+ super.testListEmptyRootDirectory();
+ break;
+ } catch (AssertionError | FileNotFoundException e) {
+ if (attempt < maxAttempts) {
+ LOG.info("Attempt {} of {} for empty root directory test failed. "
+ + "This is likely caused by eventual consistency of S3 "
+ + "listings. Attempting retry.", attempt, maxAttempts);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e2) {
+ Thread.currentThread().interrupt();
+ fail("Test interrupted.");
+ break;
+ }
+ } else {
+ LOG.error(
+ "Empty root directory test failed {} attempts. Failing test.",
+ maxAttempts);
+ throw e;
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
new file mode 100644
index 0000000..1572fbc
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractSeek.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests covering file seek.
+ */
+public class ITestS3AContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
deleted file mode 100644
index 035252b..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-
-public class TestS3AContractCreate extends AbstractContractCreateTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java
deleted file mode 100644
index 733a517..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDelete.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3AContractDelete extends AbstractContractDeleteTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java
deleted file mode 100644
index 7eb0afa..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractDistCp.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import static org.apache.hadoop.fs.s3a.Constants.MIN_MULTIPART_THRESHOLD;
-import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_SIZE;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
-
-/**
- * Contract test suite covering S3A integration with DistCp.
- */
-public class TestS3AContractDistCp extends AbstractContractDistCpTest {
-
- private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB
-
- @Override
- protected Configuration createConfiguration() {
- Configuration newConf = super.createConfiguration();
- newConf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_SETTING);
- newConf.setLong(MULTIPART_SIZE, MULTIPART_SETTING);
- return newConf;
- }
-
- @Override
- protected S3AContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
deleted file mode 100644
index 5937d49..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-
-public class TestS3AContractGetFileStatus extends AbstractContractGetFileStatusTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Override
- public void teardown() throws Exception {
- getLog().info("FS details {}", getFileSystem());
- super.teardown();
- }
-
- @Override
- protected Configuration createConfiguration() {
- Configuration conf = super.createConfiguration();
- S3ATestUtils.disableFilesystemCaching(conf);
- // aggressively low page size forces tests to go multipage
- conf.setInt(Constants.MAX_PAGING_KEYS, 2);
- return conf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java
deleted file mode 100644
index a312782..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractMkdir.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * Test dir operations on S3
- */
-public class TestS3AContractMkdir extends AbstractContractMkdirTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java
deleted file mode 100644
index f735deb..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractOpen.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3AContractOpen extends AbstractContractOpenTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java
deleted file mode 100644
index af1ed37..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRename.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.Test;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
-
-public class TestS3AContractRename extends AbstractContractRenameTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Override
- public void testRenameDirIntoExistingDir() throws Throwable {
- describe("Verify renaming a dir into an existing dir puts the files"
- +" from the source dir into the existing dir"
- +" and leaves existing files alone");
- FileSystem fs = getFileSystem();
- String sourceSubdir = "source";
- Path srcDir = path(sourceSubdir);
- Path srcFilePath = new Path(srcDir, "source-256.txt");
- byte[] srcDataset = dataset(256, 'a', 'z');
- writeDataset(fs, srcFilePath, srcDataset, srcDataset.length, 1024, false);
- Path destDir = path("dest");
-
- Path destFilePath = new Path(destDir, "dest-512.txt");
- byte[] destDateset = dataset(512, 'A', 'Z');
- writeDataset(fs, destFilePath, destDateset, destDateset.length, 1024,
- false);
- assertIsFile(destFilePath);
-
- boolean rename = fs.rename(srcDir, destDir);
- assertFalse("s3a doesn't support rename to non-empty directory", rename);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
deleted file mode 100644
index 1cf1eb8..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * root dir operations against an S3 bucket.
- */
-public class TestS3AContractRootDir extends
- AbstractContractRootDirectoryTest {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3AContractRootDir.class);
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-
- @Override
- public void testListEmptyRootDirectory() throws IOException {
- for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
- try {
- super.testListEmptyRootDirectory();
- break;
- } catch (AssertionError | FileNotFoundException e) {
- if (attempt < maxAttempts) {
- LOG.info("Attempt {} of {} for empty root directory test failed. "
- + "This is likely caused by eventual consistency of S3 "
- + "listings. Attempting retry.", attempt, maxAttempts);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e2) {
- Thread.currentThread().interrupt();
- fail("Test interrupted.");
- break;
- }
- } else {
- LOG.error(
- "Empty root directory test failed {} attempts. Failing test.",
- maxAttempts);
- throw e;
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
deleted file mode 100644
index d677ec4..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractSeek.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3a;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3AContractSeek extends AbstractContractSeekTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new S3AContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractCreate.java
new file mode 100644
index 0000000..502cf5a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractCreate.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+
+/**
+ * S3N contract tests creating files.
+ */
+public class ITestS3NContractCreate extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+
+ @Override
+ public void testOverwriteEmptyDirectory() throws Throwable {
+ ContractTestUtils.skip(
+ "blobstores can't distinguish empty directories from files");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractDelete.java
new file mode 100644
index 0000000..675f979
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractDelete.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3A contract tests covering deletes.
+ */
+public class ITestS3NContractDelete extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractMkdir.java
new file mode 100644
index 0000000..3c566f3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractMkdir.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test dir operations on S3.
+ */
+public class ITestS3NContractMkdir extends AbstractContractMkdirTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractOpen.java
new file mode 100644
index 0000000..7ebfc4e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractOpen.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3N contract tests opening files.
+ */
+public class ITestS3NContractOpen extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRename.java
new file mode 100644
index 0000000..effe9eb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRename.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3N contract tests covering rename.
+ */
+public class ITestS3NContractRename extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRootDir.java
new file mode 100644
index 0000000..3fdf868
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractRootDir.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Root dir operations against an S3 bucket.
+ */
+public class ITestS3NContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractSeek.java
new file mode 100644
index 0000000..9e1ce73
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/ITestS3NContractSeek.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.contract.s3n;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * S3N contract tests covering file seek.
+ */
+public class ITestS3NContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new NativeS3Contract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
deleted file mode 100644
index e44e2b1..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-
-public class TestS3NContractCreate extends AbstractContractCreateTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-
- @Override
- public void testOverwriteEmptyDirectory() throws Throwable {
- ContractTestUtils.skip(
- "blobstores can't distinguish empty directories from files");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
deleted file mode 100644
index 1b79d27..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractDelete extends AbstractContractDeleteTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
deleted file mode 100644
index 527a31d..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * Test dir operations on S3
- */
-public class TestS3NContractMkdir extends AbstractContractMkdirTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
deleted file mode 100644
index 2186f28..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractOpen extends AbstractContractOpenTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
deleted file mode 100644
index d673416..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractRename extends AbstractContractRenameTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
deleted file mode 100644
index 94f8483..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-/**
- * root dir operations against an S3 bucket
- */
-public class TestS3NContractRootDir extends
- AbstractContractRootDirectoryTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
deleted file mode 100644
index 6d04fff..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.contract.s3n;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-public class TestS3NContractSeek extends AbstractContractSeekTest {
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new NativeS3Contract(conf);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/52] [abbrv] hadoop git commit: YARN-3940. Application moveToQueue
should check NodeLabel permission. Contributed by Bibin A Chundatt
Posted by aw...@apache.org.
YARN-3940. Application moveToQueue should check NodeLabel permission. Contributed by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46e02ab7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46e02ab7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46e02ab7
Branch: refs/heads/HADOOP-13341
Commit: 46e02ab719d06f2708c0a61e8011b8f261235193
Parents: bfb4d95
Author: Naganarasimha <na...@apache.org>
Authored: Fri Aug 26 20:19:11 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Fri Aug 26 20:19:11 2016 +0530
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 35 ++++++
.../TestCapacitySchedulerNodeLabelUpdate.java | 116 ++++++++++++++++++-
2 files changed, 150 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e02ab7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 3b8599e..35e1147 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -94,6 +94,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdate
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -1946,6 +1947,7 @@ public class CapacityScheduler extends
LeafQueue dest = getAndCheckLeafQueue(destQueueName);
// Validation check - ACLs, submission limits for user & queue
String user = app.getUser();
+ checkQueuePartition(app, dest);
try {
dest.submitApplication(appId, user, destQueueName);
} catch (AccessControlException e) {
@@ -1971,6 +1973,39 @@ public class CapacityScheduler extends
}
/**
+ * Check application can be moved to queue with labels enabled. All labels in
+ * application life time will be checked
+ *
+ * @param appId
+ * @param dest
+ * @throws YarnException
+ */
+ private void checkQueuePartition(FiCaSchedulerApp app, LeafQueue dest)
+ throws YarnException {
+ if (!YarnConfiguration.areNodeLabelsEnabled(conf)) {
+ return;
+ }
+ Set<String> targetqueuelabels = dest.getAccessibleNodeLabels();
+ AppSchedulingInfo schedulingInfo = app.getAppSchedulingInfo();
+ Set<String> appLabelexpressions = schedulingInfo.getRequestedPartitions();
+ // default partition access always available remove empty label
+ appLabelexpressions.remove(RMNodeLabelsManager.NO_LABEL);
+ Set<String> nonAccessiblelabels = new HashSet<String>();
+ for (String label : appLabelexpressions) {
+ if (!SchedulerUtils.checkQueueLabelExpression(targetqueuelabels, label,
+ null)) {
+ nonAccessiblelabels.add(label);
+ }
+ }
+ if (nonAccessiblelabels.size() > 0) {
+ throw new YarnException(
+ "Specified queue=" + dest.getQueueName() + " can't satisfy following "
+ + "apps label expressions =" + nonAccessiblelabels
+ + " accessible node labels =" + targetqueuelabels);
+ }
+ }
+
+ /**
* Check that the String provided in input is the name of an existing,
* LeafQueue, if successful returns the queue.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e02ab7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index 9aef77c..0ae77f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import static org.junit.Assert.fail;
+
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Map;
@@ -31,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -86,7 +89,53 @@ public class TestCapacitySchedulerNodeLabelUpdate {
return conf;
}
-
+
+ private Configuration getConfigurationWithSubQueueLabels(
+ Configuration config) {
+ CapacitySchedulerConfiguration conf2 =
+ new CapacitySchedulerConfiguration(config);
+
+ // Define top-level queues
+ conf2.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] {"a", "b"});
+ conf2.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ conf2.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
+
+ final String a = CapacitySchedulerConfiguration.ROOT + ".a";
+ final String b = CapacitySchedulerConfiguration.ROOT + ".b";
+ final String aa1 = a + ".a1";
+ final String aa2 = a + ".a2";
+ final String aa3 = a + ".a3";
+ final String aa4 = a + ".a4";
+ conf2.setQueues(a, new String[] {"a1", "a2", "a3", "a4"});
+ conf2.setCapacity(a, 50);
+ conf2.setCapacity(b, 50);
+ conf2.setCapacity(aa1, 40);
+ conf2.setCapacity(aa2, 20);
+ conf2.setCapacity(aa3, 20);
+ conf2.setCapacity(aa4, 20);
+ conf2.setAccessibleNodeLabels(a, ImmutableSet.of("x", "y", "z"));
+ conf2.setAccessibleNodeLabels(aa1, ImmutableSet.of("x", "y"));
+ conf2.setAccessibleNodeLabels(aa2, ImmutableSet.of("y"));
+ conf2.setAccessibleNodeLabels(aa3, ImmutableSet.of("x", "y", "z"));
+ conf2.setAccessibleNodeLabels(aa4, ImmutableSet.of("x", "y"));
+ conf2.setCapacityByLabel(a, "x", 50);
+ conf2.setCapacityByLabel(a, "y", 50);
+ conf2.setCapacityByLabel(a, "z", 50);
+ conf2.setCapacityByLabel(b, "x", 50);
+ conf2.setCapacityByLabel(b, "y", 50);
+ conf2.setCapacityByLabel(b, "z", 50);
+ conf2.setCapacityByLabel(aa1, "x", 50);
+ conf2.setCapacityByLabel(aa3, "x", 25);
+ conf2.setCapacityByLabel(aa4, "x", 25);
+ conf2.setCapacityByLabel(aa1, "y", 25);
+ conf2.setCapacityByLabel(aa2, "y", 25);
+ conf2.setCapacityByLabel(aa4, "y", 50);
+ conf2.setCapacityByLabel(aa3, "z", 50);
+ conf2.setCapacityByLabel(aa4, "z", 50);
+ return conf2;
+ }
+
private Set<String> toSet(String... elements) {
Set<String> set = Sets.newHashSet(elements);
return set;
@@ -364,6 +413,71 @@ public class TestCapacitySchedulerNodeLabelUpdate {
rm.close();
}
+ @Test(timeout = 3000000)
+ public void testMoveApplicationWithLabel() throws Exception {
+ // set node -> label
+ mgr.addToCluserNodeLabelsWithDefaultExclusivity(
+ ImmutableSet.of("x", "y", "z"));
+ // set mapping:
+ // h1 -> x
+ // h2 -> y
+ mgr.addLabelsToNode(
+ ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+ mgr.addLabelsToNode(
+ ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
+ mgr.addLabelsToNode(
+ ImmutableMap.of(NodeId.newInstance("h4", 0), toSet("z")));
+ // inject node label manager
+ MockRM rm = new MockRM(getConfigurationWithSubQueueLabels(conf)) {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm.getRMContext().getContainerTokenSecretManager().rollMasterKey();
+ rm.getRMContext().getNMTokenSecretManager().rollMasterKey();
+ rm.getRMContext().setNodeLabelManager(mgr);
+ rm.start();
+ MockNM nm1 = rm.registerNode("h1:1234", 4096 * 2);
+ MockNM nm2 = rm.registerNode("h2:1234", 4096 * 2);
+ MockNM nm3 = rm.registerNode("h3:1234", 4096 * 2);
+ MockNM nm4 = rm.registerNode("h4:1234", 4096 * 2);
+ // launch an app to queue a1 (label = x), and check all container will
+ // be allocated in h1
+ RMApp app1 = rm.submitApp(GB, "app", "user", null, "a1");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
+ am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
+ ContainerId container1 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ rm.waitForState(nm1, container1, RMContainerState.ALLOCATED, 10 * 1000);
+ am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "y");
+ ContainerId container2 =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
+ rm.waitForState(nm2, container2, RMContainerState.ALLOCATED, 10 * 1000);
+ CapacityScheduler scheduler =
+ ((CapacityScheduler) rm.getResourceScheduler());
+ try {
+ scheduler.moveApplication(app1.getApplicationId(), "a2");
+ fail("Should throw exception since target queue doesnt have "
+ + "required labels");
+ } catch (Exception e) {
+ Assert.assertTrue("Yarn Exception should be thrown",
+ e instanceof YarnException);
+ Assert.assertEquals("Specified queue=a2 can't satisfy "
+ + "following apps label expressions =[x] accessible "
+ + "node labels =[y]", e.getMessage());
+ }
+ try {
+ scheduler.moveApplication(app1.getApplicationId(), "a3");
+ scheduler.moveApplication(app1.getApplicationId(), "a4");
+ // Check move to queue with accessible label ANY
+ scheduler.moveApplication(app1.getApplicationId(), "b");
+ } catch (Exception e) {
+ fail("Should not throw exception since target queue has "
+ + "required labels");
+ }
+ rm.stop();
+ }
@Test (timeout = 60000)
public void testComplexResourceUsageWhenNodeUpdatesPartition()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[51/52] [abbrv] hadoop git commit: YARN-5373. NPE listing wildcard
directory in containerLaunch. (Daniel Templeton via kasha)
Posted by aw...@apache.org.
YARN-5373. NPE listing wildcard directory in containerLaunch. (Daniel Templeton via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cde3a005
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cde3a005
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cde3a005
Branch: refs/heads/HADOOP-13341
Commit: cde3a00526c562a500308232e2b93498d22c90d7
Parents: 9ef632f
Author: Karthik Kambatla <ka...@apache.org>
Authored: Fri Aug 26 11:04:33 2016 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Fri Aug 26 11:04:33 2016 -0700
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 42 +++--
.../nodemanager/DockerContainerExecutor.java | 4 +-
.../nodemanager/LinuxContainerExecutor.java | 40 ++++-
.../launcher/ContainerLaunch.java | 2 +-
.../linux/privileged/PrivilegedOperation.java | 6 +-
.../impl/container-executor.c | 54 ++++++-
.../impl/container-executor.h | 10 +-
.../main/native/container-executor/impl/main.c | 40 +++--
.../test/test-container-executor.c | 157 +++++++++++++++++++
.../launcher/TestContainerLaunch.java | 12 +-
10 files changed, 325 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index c73c4c7..818b0ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -67,8 +67,8 @@ import org.apache.hadoop.util.StringUtils;
* underlying OS. All executor implementations must extend ContainerExecutor.
*/
public abstract class ContainerExecutor implements Configurable {
- private static final String WILDCARD = "*";
private static final Log LOG = LogFactory.getLog(ContainerExecutor.class);
+ protected static final String WILDCARD = "*";
/**
* The permissions to use when creating the launch script.
@@ -274,15 +274,16 @@ public abstract class ContainerExecutor implements Configurable {
* @param environment the environment variables and their values
* @param resources the resources which have been localized for this
* container. Symlinks will be created to these localized resources
- * @param command the command that will be run.
- * @param logDir the log dir to copy debugging information to
+ * @param command the command that will be run
+ * @param logDir the log dir to which to copy debugging information
+ * @param user the username of the job owner
* @throws IOException if any errors happened writing to the OutputStream,
* while creating symlinks
*/
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
- Map<Path, List<String>> resources, List<String> command, Path logDir)
- throws IOException {
- this.writeLaunchEnv(out, environment, resources, command, logDir,
+ Map<Path, List<String>> resources, List<String> command, Path logDir,
+ String user) throws IOException {
+ this.writeLaunchEnv(out, environment, resources, command, logDir, user,
ContainerLaunch.CONTAINER_SCRIPT);
}
@@ -295,17 +296,17 @@ public abstract class ContainerExecutor implements Configurable {
* @param environment the environment variables and their values
* @param resources the resources which have been localized for this
* container. Symlinks will be created to these localized resources
- * @param command the command that will be run.
- * @param logDir the log dir to copy debugging information to
+ * @param command the command that will be run
+ * @param logDir the log dir to which to copy debugging information
+ * @param user the username of the job owner
* @param outFilename the path to which to write the launch environment
* @throws IOException if any errors happened writing to the OutputStream,
* while creating symlinks
*/
@VisibleForTesting
- public void writeLaunchEnv(OutputStream out,
- Map<String, String> environment, Map<Path, List<String>> resources,
- List<String> command, Path logDir, String outFilename)
- throws IOException {
+ public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
+ Map<Path, List<String>> resources, List<String> command, Path logDir,
+ String user, String outFilename) throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
Set<String> whitelist = new HashSet<>();
@@ -334,9 +335,7 @@ public abstract class ContainerExecutor implements Configurable {
if (new Path(linkName).getName().equals(WILDCARD)) {
// If this is a wildcarded path, link to everything in the
// directory from the working directory
- File directory = new File(resourceEntry.getKey().toString());
-
- for (File wildLink : directory.listFiles()) {
+ for (File wildLink : readDirAsUser(user, resourceEntry.getKey())) {
sb.symlink(new Path(wildLink.toString()),
new Path(wildLink.getName()));
}
@@ -371,6 +370,19 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
+ * Return the files in the target directory. If retrieving the list of files
+ * requires specific access rights, that access will happen as the
+ * specified user. The list will not include entries for "." or "..".
+ *
+ * @param user the user as whom to access the target directory
+ * @param dir the target directory
+ * @return a list of files in the target directory
+ */
+ protected File[] readDirAsUser(String user, Path dir) {
+ return new File(dir.toString()).listFiles();
+ }
+
+ /**
* The container exit code.
*/
public enum ExitCode {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index 7bf6989..ebf9566 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -330,8 +330,8 @@ public class DockerContainerExecutor extends ContainerExecutor {
* the docker image and write them out to an OutputStream.
*/
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
- Map<Path, List<String>> resources, List<String> command, Path logDir)
- throws IOException {
+ Map<Path, List<String>> resources, List<String> command, Path logDir,
+ String user) throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 6c658bc..6890b25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler;
import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler;
-
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -645,6 +644,45 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ protected File[] readDirAsUser(String user, Path dir) {
+ List<File> files = new ArrayList<>();
+ PrivilegedOperation listAsUserOp = new PrivilegedOperation(
+ PrivilegedOperation.OperationType.LIST_AS_USER, (String)null);
+ String runAsUser = getRunAsUser(user);
+ String dirString = "";
+
+ if (dir != null) {
+ dirString = dir.toUri().getPath();
+ }
+
+ listAsUserOp.appendArgs(runAsUser, user,
+ Integer.toString(
+ PrivilegedOperation.RunAsUserCommand.LIST_AS_USER.getValue()),
+ dirString);
+
+ try {
+ PrivilegedOperationExecutor privOpExecutor =
+ PrivilegedOperationExecutor.getInstance(super.getConf());
+
+ String results =
+ privOpExecutor.executePrivilegedOperation(listAsUserOp, true);
+
+ for (String file: results.split("\n")) {
+ // The container-executor always dumps its log output to stdout, which
+ // includes 3 lines that start with "main : "
+ if (!file.startsWith("main :")) {
+ files.add(new File(new File(dirString), file));
+ }
+ }
+ } catch (PrivilegedOperationException e) {
+ LOG.error("ListAsUser for " + dir + " returned with exit code: "
+ + e.getExitCode(), e);
+ }
+
+ return files.toArray(new File[files.size()]);
+ }
+
+ @Override
public boolean isContainerAlive(ContainerLivenessContext ctx)
throws IOException {
String user = ctx.getUser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 81b6c1f..14190fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -267,7 +267,7 @@ public class ContainerLaunch implements Callable<Integer> {
// Write out the environment
exec.writeLaunchEnv(containerScriptOutStream, environment,
localResources, launchContext.getCommands(),
- new Path(containerLogDirs.get(0)));
+ new Path(containerLogDirs.get(0)), user);
// /////////// End of writing out container-script
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
index 259dee8..8402a16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java
@@ -50,7 +50,8 @@ public class PrivilegedOperation {
TC_READ_STATE("--tc-read-state"),
TC_READ_STATS("--tc-read-stats"),
ADD_PID_TO_CGROUP(""), //no CLI switch supported yet.
- RUN_DOCKER_CMD("--run-docker");
+ RUN_DOCKER_CMD("--run-docker"),
+ LIST_AS_USER(""); //no CLI switch supported yet.
private final String option;
@@ -146,7 +147,8 @@ public class PrivilegedOperation {
LAUNCH_CONTAINER(1),
SIGNAL_CONTAINER(2),
DELETE_AS_USER(3),
- LAUNCH_DOCKER_CONTAINER(4);
+ LAUNCH_DOCKER_CONTAINER(4),
+ LIST_AS_USER(5);
private int value;
RunAsUserCommand(int value) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index f8c7f6e..ca3847e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -439,7 +439,7 @@ char *concatenate(char *concat_pattern, char *return_path_name,
for (j = 0; j < numArgs; j++) {
arg = va_arg(ap, char*);
if (arg == NULL) {
- fprintf(LOGFILE, "One of the arguments passed for %s in null.\n",
+ fprintf(LOGFILE, "One of the arguments passed for %s is null.\n",
return_path_name);
return NULL;
}
@@ -1929,6 +1929,58 @@ int delete_as_user(const char *user,
return ret;
}
+/**
+ * List the files in the given directory as the user.
+ * user: the user listing the files
+ * target_dir: the directory from which to list files
+ */
+int list_as_user(const char *target_dir) {
+ int ret = 0;
+ struct stat sb;
+
+ if (stat(target_dir, &sb) != 0) {
+ // If directory doesn't exist or can't be accessed, error out
+ fprintf(LOGFILE, "Could not stat %s - %s\n", target_dir,
+ strerror(errno));
+ ret = -1;
+ } else if (!S_ISDIR(sb.st_mode)) {
+ // If it's not a directory, list it as the only file
+ printf("%s\n", target_dir);
+ } else {
+ DIR *dir = opendir(target_dir);
+
+ if (dir != NULL) {
+ struct dirent *file;
+
+ errno = 0;
+
+ do {
+ file = readdir(dir);
+
+ // Ignore the . and .. entries
+ if ((file != NULL) &&
+ (strcmp(".", file->d_name) != 0) &&
+ (strcmp("..", file->d_name) != 0)) {
+ printf("%s\n", file->d_name);
+ }
+ } while (file != NULL);
+
+ // If we ended the directory read early on an error, then error out
+ if (errno != 0) {
+ fprintf(LOGFILE, "Could not read directory %s - %s\n", target_dir,
+ strerror(errno));
+ ret = -1;
+ }
+ } else {
+ fprintf(LOGFILE, "Could not open directory %s - %s\n", target_dir,
+ strerror(errno));
+ ret = -1;
+ }
+ }
+
+ return ret;
+}
+
void chown_dir_contents(const char *dir_path, uid_t uid, gid_t gid) {
DIR *dp;
struct dirent *ep;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index be78283..1858555 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -31,7 +31,8 @@ enum command {
LAUNCH_CONTAINER = 1,
SIGNAL_CONTAINER = 2,
DELETE_AS_USER = 3,
- LAUNCH_DOCKER_CONTAINER = 4
+ LAUNCH_DOCKER_CONTAINER = 4,
+ LIST_AS_USER = 5
};
enum errorcodes {
@@ -79,7 +80,8 @@ enum operations {
RUN_AS_USER_SIGNAL_CONTAINER = 8,
RUN_AS_USER_DELETE = 9,
RUN_AS_USER_LAUNCH_DOCKER_CONTAINER = 10,
- RUN_DOCKER = 11
+ RUN_DOCKER = 11,
+ RUN_AS_USER_LIST = 12
};
#define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group"
@@ -189,6 +191,10 @@ int delete_as_user(const char *user,
const char *dir_to_be_deleted,
char* const* baseDirs);
+// List the files in the given directory on stdout. The target_dir is always
+// assumed to be an absolute path.
+int list_as_user(const char *target_dir);
+
// set the uid and gid of the node manager. This is used when doing some
// priviledged operations for setting the effective uid and gid.
void set_nm_uid(uid_t user, gid_t group);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 5fc92d6..56215ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -58,11 +58,12 @@ static void display_usage(FILE *stream) {
" launch docker container: %2d appid containerid workdir container-script " \
"tokens pidfile nm-local-dirs nm-log-dirs docker-command-file resources optional-tc-command-file\n" \
" signal container: %2d container-pid signal\n" \
- " delete as user: %2d relative-path\n" ;
+ " delete as user: %2d relative-path\n" \
+ " list as user: %2d relative-path\n" ;
fprintf(stream, usage_template, INITIALIZE_CONTAINER, LAUNCH_CONTAINER, LAUNCH_DOCKER_CONTAINER,
- SIGNAL_CONTAINER, DELETE_AS_USER);
+ SIGNAL_CONTAINER, DELETE_AS_USER, LIST_AS_USER);
}
/* Sets up log files for normal/error logging */
@@ -169,20 +170,20 @@ static void assert_valid_setup(char *argv0) {
static struct {
char *cgroups_hierarchy;
char *traffic_control_command_file;
- const char * run_as_user_name;
- const char * yarn_user_name;
+ const char *run_as_user_name;
+ const char *yarn_user_name;
char *local_dirs;
char *log_dirs;
char *resources_key;
char *resources_value;
char **resources_values;
- const char * app_id;
- const char * container_id;
- const char * cred_file;
- const char * script_file;
- const char * current_dir;
- const char * pid_file;
- const char *dir_to_be_deleted;
+ const char *app_id;
+ const char *container_id;
+ const char *cred_file;
+ const char *script_file;
+ const char *current_dir;
+ const char *pid_file;
+ const char *target_dir;
int container_pid;
int signal;
const char *docker_command_file;
@@ -417,9 +418,13 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
return 0;
case DELETE_AS_USER:
- cmd_input.dir_to_be_deleted = argv[optind++];
+ cmd_input.target_dir = argv[optind++];
*operation = RUN_AS_USER_DELETE;
return 0;
+ case LIST_AS_USER:
+ cmd_input.target_dir = argv[optind++];
+ *operation = RUN_AS_USER_LIST;
+ return 0;
default:
fprintf(ERRORFILE, "Invalid command %d not supported.",command);
fflush(ERRORFILE);
@@ -554,9 +559,18 @@ int main(int argc, char **argv) {
}
exit_code = delete_as_user(cmd_input.yarn_user_name,
- cmd_input.dir_to_be_deleted,
+ cmd_input.target_dir,
argv + optind);
break;
+ case RUN_AS_USER_LIST:
+ exit_code = set_user(cmd_input.run_as_user_name);
+
+ if (exit_code != 0) {
+ break;
+ }
+
+ exit_code = list_as_user(cmd_input.target_dir);
+ break;
}
flush_and_close_log_files();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index cad7c3f..f174a9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -454,6 +454,163 @@ void test_delete_user() {
free(app_dir);
}
+/**
+ * Read a file and tokenize it on newlines. Place up to max lines into lines.
+ * The max+1st element of lines will be set to NULL.
+ *
+ * @param file the name of the file to open
+ * @param lines the pointer array into which to place the lines
+ * @param max the max number of lines to add to lines
+ */
+void read_lines(const char* file, char **lines, size_t max) {
+ char buf[4096];
+ size_t nread;
+
+ int fd = open(file, O_RDONLY);
+
+ if (fd < 0) {
+ printf("FAIL: failed to open directory listing file: %s\n", file);
+ exit(1);
+ } else {
+ char *cur = buf;
+ size_t count = sizeof buf;
+
+ while ((nread = read(fd, cur, count)) > 0) {
+ cur += nread;
+ count -= nread;
+ }
+
+ if (nread < 0) {
+ printf("FAIL: failed to read directory listing file: %s\n", file);
+ exit(1);
+ }
+
+ close(fd);
+ }
+
+ char* entity = strtok(buf, "\n");
+ int i;
+
+ for (i = 0; i < max; i++) {
+ if (entity == NULL) {
+ break;
+ }
+
+ lines[i] = (char *)malloc(sizeof(char) * (strlen(entity) + 1));
+ strcpy(lines[i], entity);
+ entity = strtok(NULL, "\n");
+ }
+
+ lines[i] = NULL;
+}
+
+void test_list_as_user() {
+ printf("\nTesting list_as_user\n");
+ char buffer[4096];
+
+ char *app_dir =
+ get_app_directory(TEST_ROOT "/local-1", "yarn", "app_4");
+
+ if (mkdirs(app_dir, 0700) != 0) {
+ printf("FAIL: unble to create application directory: %s\n", app_dir);
+ exit(1);
+ }
+
+ // Test with empty dir string
+ sprintf(buffer, "");
+ int ret = list_as_user(buffer);
+
+ if (ret == 0) {
+ printf("FAIL: did not fail on empty directory string\n");
+ exit(1);
+ }
+
+ // Test with a non-existent directory
+ sprintf(buffer, "%s/output", app_dir);
+
+ ret = list_as_user(buffer);
+
+ if (ret == 0) {
+ printf("FAIL: did not fail on non-existent directory\n");
+ exit(1);
+ }
+
+ // Write a couple files to list
+ sprintf(buffer, "%s/file1", app_dir);
+
+ if (write_config_file(buffer, 1) != 0) {
+ exit(1);
+ }
+
+ sprintf(buffer, "%s/.file2", app_dir);
+
+ if (write_config_file(buffer, 1) != 0) {
+ exit(1);
+ }
+
+ // Also create a directory
+ sprintf(buffer, "%s/output", app_dir);
+
+ if (mkdirs(buffer, 0700) != 0) {
+ exit(1);
+ }
+
+ // Test the regular case
+ // Store a copy of stdout, then redirect it to a file
+ sprintf(buffer, "%s/output/files", app_dir);
+
+ int oldout = dup(STDOUT_FILENO);
+ int fd = open(buffer, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
+
+ dup2(fd, STDOUT_FILENO);
+
+ // Now list the files
+ ret = list_as_user(app_dir);
+
+ if (ret != 0) {
+ printf("FAIL: unable to list files in regular case\n");
+ exit(1);
+ }
+
+ // Restore stdout
+ close(fd);
+ dup2(oldout, STDOUT_FILENO);
+
+ // Check the output -- shouldn't be more than a couple lines
+ char *lines[16];
+
+ read_lines(buffer, lines, 15);
+
+ int got_file1 = 0;
+ int got_file2 = 0;
+ int got_output = 0;
+ int i;
+
+ for (i = 0; i < sizeof lines; i++) {
+ if (lines[i] == NULL) {
+ break;
+ } else if (strcmp("file1", lines[i]) == 0) {
+ got_file1 = 1;
+ } else if (strcmp(".file2", lines[i]) == 0) {
+ got_file2 = 1;
+ } else if (strcmp("output", lines[i]) == 0) {
+ got_output = 1;
+ } else {
+ printf("FAIL: listed extraneous file: %s\n", lines[i]);
+ exit(1);
+ }
+
+ free(lines[i]);
+ }
+
+ if (!got_file1 || !got_file2 || !got_output) {
+ printf("FAIL: missing files in listing\n");
+ exit(1);
+ }
+
+ free(app_dir);
+}
+
void run_test_in_child(const char* test_name, void (*func)()) {
printf("\nRunning test %s in child process\n", test_name);
fflush(stdout);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde3a005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index a06822a..be6eadb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -174,7 +174,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
new DefaultContainerExecutor()
.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), tempFile.getName());
+ new Path(localLogDir.getAbsolutePath()), "user",
+ tempFile.getName());
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -243,7 +244,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
new DefaultContainerExecutor()
.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()));
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -298,7 +299,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
List<String> commands = new ArrayList<String>();
new DefaultContainerExecutor()
.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()));
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
@@ -377,7 +378,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
commands.add(command);
ContainerExecutor exec = new DefaultContainerExecutor();
exec.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()));
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
@@ -1400,7 +1401,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
ContainerExecutor exec = new DefaultContainerExecutor();
exec.setConf(conf);
exec.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), tempFile.getName());
+ new Path(localLogDir.getAbsolutePath()), "user",
+ tempFile.getName());
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/52] [abbrv] hadoop git commit: HDFS-10783. The option '-maxSize'
and '-step' fail in OfflineImageViewer. Contributed by Yiqun Lin.
Posted by aw...@apache.org.
HDFS-10783. The option '-maxSize' and '-step' fail in OfflineImageViewer. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e90f3359
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e90f3359
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e90f3359
Branch: refs/heads/HADOOP-13341
Commit: e90f3359de299ef5e3a54ca71070e3dfe1dbb98c
Parents: 0d5997d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Aug 23 19:56:27 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Aug 23 19:57:23 2016 +0900
----------------------------------------------------------------------
.../offlineImageViewer/OfflineImageViewer.java | 2 ++
.../TestOfflineImageViewer.java | 19 +++++++++++++++++++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e90f3359/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
index 7f81ba8..770cde1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
@@ -186,6 +186,8 @@ public class OfflineImageViewer {
options.addOption("p", "processor", true, "");
options.addOption("h", "help", false, "");
+ options.addOption("maxSize", true, "");
+ options.addOption("step", true, "");
options.addOption("skipBlocks", false, "");
options.addOption("printToScreen", false, "");
options.addOption("delimiter", true, "");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e90f3359/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index b9aa7f3..a7c30ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -601,4 +601,23 @@ public class TestOfflineImageViewer {
"FileDistribution", "-maxSize", "23", "-step", "4"});
assertEquals(0, status);
}
+
+ @Test
+ public void testOfflineImageViewerMaxSizeAndStepOptions() throws Exception {
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ final PrintStream out = new PrintStream(bytes);
+ final PrintStream oldOut = System.out;
+ try {
+ System.setOut(out);
+ // Add the -h option to make the test only for option parsing,
+ // and don't need to do the following operations.
+ OfflineImageViewer.main(new String[] {"-i", "-", "-o", "-", "-p",
+ "FileDistribution", "-maxSize", "512", "-step", "8", "-h"});
+ Assert.assertFalse(bytes.toString().contains(
+ "Error parsing command-line options: "));
+ } finally {
+ System.setOut(oldOut);
+ IOUtils.closeStream(out);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/52] [abbrv] hadoop git commit: HADOOP-13497. fix wrong command in
CredentialProviderAPI.md. Contributed by Yuanbo Liu.
Posted by aw...@apache.org.
HADOOP-13497. fix wrong command in CredentialProviderAPI.md. Contributed by Yuanbo Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aae8d6b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aae8d6b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aae8d6b
Branch: refs/heads/HADOOP-13341
Commit: 8aae8d6bf03ade0607547ed461dc99a336a7e9d4
Parents: 126d165
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Wed Aug 24 00:03:27 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Wed Aug 24 00:03:27 2016 +0900
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/CredentialProviderAPI.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aae8d6b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
index 1142372..a40bf2b 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
@@ -119,7 +119,7 @@ See the command options detail in the [Commands Manual](CommandsManual.html#cred
Utilizing the credential command will often be for provisioning a password or secret to a particular credential store provider. In order to explicitly indicate which provider store to use the `-provider` option should be used.
-Example: `hadoop credential create ssl.server.keystore.password jceks://file/tmp/test.jceks`
+Example: `hadoop credential create ssl.server.keystore.password -provider jceks://file/tmp/test.jceks`
In order to indicate a particular provider type and location, the user must provide the `hadoop.security.credential.provider.path` configuration element in core-site.xml or use the command line option `-provider` on each of the credential management commands. This provider path is a comma-separated list of URLs that indicates the type and location of a list of providers that should be consulted. For example, the following path: `user:///,jceks://file/tmp/test.jceks,jceks://hdfs@nn1.example.com/my/path/test.jceks` indicates that the current user's credentials file should be consulted through the User Provider, that the local file located at `/tmp/test.jceks` is a Java Keystore Provider and that the file located within HDFS at `nn1.example.com/my/path/test.jceks` is also a store for a Java Keystore Provider.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/52] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by aw...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java
deleted file mode 100644
index 34d78a5..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import org.apache.hadoop.fs.TestFileContext;
-
-/**
- * Implementation of TestFileContext for S3a
- */
-public class TestS3AFileContext extends TestFileContext{
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java
deleted file mode 100644
index b0c4d84..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-
-/**
- * Extends FileContextCreateMkdirBaseTest for a S3a FileContext
- */
-public class TestS3AFileContextCreateMkdir
- extends FileContextCreateMkdirBaseTest {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- super.setUp();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java
deleted file mode 100644
index 4d200d1..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * S3A implementation of FileContextMainOperationsBaseTest
- */
-public class TestS3AFileContextMainOperations
- extends FileContextMainOperationsBaseTest {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- super.setUp();
- }
-
- @Override
- protected boolean listCorruptedBlocksSupported() {
- return false;
- }
-
- @Test
- @Ignore
- public void testCreateFlagAppendExistingFile() throws IOException {
- //append not supported, so test removed
- }
-
- @Test
- @Ignore
- public void testCreateFlagCreateAppendExistingFile() throws IOException {
- //append not supported, so test removed
- }
-
- @Test
- @Ignore
- public void testSetVerifyChecksum() throws IOException {
- //checksums ignored, so test removed
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java
deleted file mode 100644
index a9f4848..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.net.URI;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FCStatisticsBaseTest;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-
-/**
- * S3a implementation of FCStatisticsBaseTest
- */
-public class TestS3AFileContextStatistics extends FCStatisticsBaseTest {
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true);
- }
-
- @After
- public void tearDown() throws Exception {
- fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true);
- }
-
- @Override
- protected void verifyReadBytes(FileSystem.Statistics stats) {
- // one blockSize for read, one for pread
- Assert.assertEquals(2 * blockSize, stats.getBytesRead());
- }
-
- @Override
- protected void verifyWrittenBytes(FileSystem.Statistics stats) {
- //No extra bytes are written
- Assert.assertEquals(blockSize, stats.getBytesWritten());
- }
-
- @Override
- protected URI getFsUri() {
- return fc.getHomeDirectory().toUri();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java
deleted file mode 100644
index 3da7b19..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextURIBase;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * S3a implementation of FileContextURIBase
- */
-public class TestS3AFileContextURI extends FileContextURIBase {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc1 = S3ATestUtils.createTestFileContext(conf);
- fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
- super.setUp();
- }
-
- @Test
- @Ignore
- public void testFileStatus() throws IOException {
- //test disabled (the statistics tested with this method are not relevant for an S3FS)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java
deleted file mode 100644
index 666f4c2..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.fileContext;
-
-import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContextUtilBase;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.junit.Before;
-
-/**
- * S3A implementation of FileContextUtilBase
- */
-public class TestS3AFileContextUtil extends FileContextUtilBase {
-
- @Before
- public void setUp() throws IOException, Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- super.setUp();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java
new file mode 100644
index 0000000..a375664
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Tests file deletion with multi-delete disabled.
+ */
+public class ITestS3ADeleteFilesOneByOne extends ITestS3ADeleteManyFiles {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration configuration = super.createConfiguration();
+ configuration.setBoolean(Constants.ENABLE_MULTI_DELETE, false);
+ return configuration;
+ }
+
+ @Override
+ @Test
+ public void testOpenCreate() throws IOException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
new file mode 100644
index 0000000..208c491
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * Test some scalable operations related to file renaming and deletion.
+ */
+public class ITestS3ADeleteManyFiles extends S3AScaleTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3ADeleteManyFiles.class);
+
+ /**
+ * CAUTION: If this test starts failing, please make sure that the
+ * {@link org.apache.hadoop.fs.s3a.Constants#MAX_THREADS} configuration is not
+ * set too low. Alternatively, consider reducing the
+ * <code>scale.test.operation.count</code> parameter in
+ * <code>getOperationCount()</code>.
+ *
+ * @see #getOperationCount()
+ */
+ @Test
+ public void testBulkRenameAndDelete() throws Throwable {
+ final Path scaleTestDir = getTestPath();
+ final Path srcDir = new Path(scaleTestDir, "src");
+ final Path finalDir = new Path(scaleTestDir, "final");
+ final long count = getOperationCount();
+ ContractTestUtils.rm(fs, scaleTestDir, true, false);
+
+ fs.mkdirs(srcDir);
+ fs.mkdirs(finalDir);
+
+ int testBufferSize = fs.getConf()
+ .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
+ ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
+ // use Executor to speed up file creation
+ ExecutorService exec = Executors.newFixedThreadPool(16);
+ final ExecutorCompletionService<Boolean> completionService =
+ new ExecutorCompletionService<>(exec);
+ try {
+ final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');
+
+ for (int i = 0; i < count; ++i) {
+ final String fileName = "foo-" + i;
+ completionService.submit(new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws IOException {
+ ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
+ false, data);
+ return fs.exists(new Path(srcDir, fileName));
+ }
+ });
+ }
+ for (int i = 0; i < count; ++i) {
+ final Future<Boolean> future = completionService.take();
+ try {
+ if (!future.get()) {
+ LOG.warn("cannot create file");
+ }
+ } catch (ExecutionException e) {
+ LOG.warn("Error while uploading file", e.getCause());
+ throw e;
+ }
+ }
+ } finally {
+ exec.shutdown();
+ }
+
+ int nSrcFiles = fs.listStatus(srcDir).length;
+ fs.rename(srcDir, finalDir);
+ assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
+ new Path(srcDir, "foo-" + 0));
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
+ new Path(srcDir, "foo-" + count / 2));
+ ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
+ new Path(srcDir, "foo-" + (count - 1)));
+ ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
+ new Path(finalDir, "foo-" + 0));
+ ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
+ new Path(finalDir, "foo-" + count/2));
+ ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
+ new Path(finalDir, "foo-" + (count-1)));
+
+ ContractTestUtils.assertDeleted(fs, finalDir, true, false);
+ }
+
+ @Test
+ public void testOpenCreate() throws IOException {
+ Path dir = new Path("/tests3a");
+ ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
+ ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
+ ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);
+
+
+ /*
+ Enable to test the multipart upload
+ try {
+ ContractTestUtils.createAndVerifyFile(fs, dir,
+ (long)6 * 1024 * 1024 * 1024);
+ } catch (IOException e) {
+ fail(e.getMessage());
+ }
+ */
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
new file mode 100644
index 0000000..b5f4eb3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Statistic;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.s3a.Statistic.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+
+/**
+ * Test the performance of listing files/directories.
+ */
+public class ITestS3ADirectoryPerformance extends S3AScaleTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ITestS3ADirectoryPerformance.class);
+
+ @Test
+ public void testListOperations() throws Throwable {
+ describe("Test recursive list operations");
+ final Path scaleTestDir = getTestPath();
+ final Path listDir = new Path(scaleTestDir, "lists");
+
+ // scale factor.
+ int scale = getConf().getInt(KEY_DIRECTORY_COUNT, DEFAULT_DIRECTORY_COUNT);
+ int width = scale;
+ int depth = scale;
+ int files = scale;
+ MetricDiff metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
+ MetricDiff listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
+ MetricDiff listContinueRequests =
+ new MetricDiff(fs, OBJECT_CONTINUE_LIST_REQUESTS);
+ MetricDiff listStatusCalls = new MetricDiff(fs, INVOCATION_LIST_FILES);
+ MetricDiff getFileStatusCalls =
+ new MetricDiff(fs, INVOCATION_GET_FILE_STATUS);
+ NanoTimer createTimer = new NanoTimer();
+ TreeScanResults created =
+ createSubdirs(fs, listDir, depth, width, files, 0);
+ // add some empty directories
+ int emptyDepth = 1 * scale;
+ int emptyWidth = 3 * scale;
+
+ created.add(createSubdirs(fs, listDir, emptyDepth, emptyWidth, 0,
+ 0, "empty", "f-", ""));
+ createTimer.end("Time to create %s", created);
+ LOG.info("Time per operation: {}",
+ toHuman(createTimer.nanosPerOperation(created.totalCount())));
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+
+ describe("Listing files via treewalk");
+ try {
+ // Scan the directory via an explicit tree walk.
+ // This is the baseline for any listing speedups.
+ NanoTimer treeWalkTimer = new NanoTimer();
+ TreeScanResults treewalkResults = treeWalk(fs, listDir);
+ treeWalkTimer.end("List status via treewalk of %s", created);
+
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+ assertEquals("Files found in listFiles(recursive=true) " +
+ " created=" + created + " listed=" + treewalkResults,
+ created.getFileCount(), treewalkResults.getFileCount());
+
+ describe("Listing files via listFiles(recursive=true)");
+ // listFiles() does the recursion internally
+ NanoTimer listFilesRecursiveTimer = new NanoTimer();
+
+ TreeScanResults listFilesResults = new TreeScanResults(
+ fs.listFiles(listDir, true));
+
+ listFilesRecursiveTimer.end("listFiles(recursive=true) of %s", created);
+ assertEquals("Files found in listFiles(recursive=true) " +
+ " created=" + created + " listed=" + listFilesResults,
+ created.getFileCount(), listFilesResults.getFileCount());
+
+ // only two list operations should have taken place
+ print(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+ assertEquals(listRequests.toString(), 2, listRequests.diff());
+ reset(metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+
+
+ } finally {
+ describe("deletion");
+ // deletion at the end of the run
+ NanoTimer deleteTimer = new NanoTimer();
+ fs.delete(listDir, true);
+ deleteTimer.end("Deleting directory tree");
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+ }
+ }
+
+ @Test
+ public void testTimeToStatEmptyDirectory() throws Throwable {
+ describe("Time to stat an empty directory");
+ Path path = new Path(getTestPath(), "empty");
+ fs.mkdirs(path);
+ timeToStatPath(path);
+ }
+
+ @Test
+ public void testTimeToStatNonEmptyDirectory() throws Throwable {
+ describe("Time to stat a non-empty directory");
+ Path path = new Path(getTestPath(), "dir");
+ fs.mkdirs(path);
+ touch(fs, new Path(path, "file"));
+ timeToStatPath(path);
+ }
+
+ @Test
+ public void testTimeToStatFile() throws Throwable {
+ describe("Time to stat a simple file");
+ Path path = new Path(getTestPath(), "file");
+ touch(fs, path);
+ timeToStatPath(path);
+ }
+
+ @Test
+ public void testTimeToStatRoot() throws Throwable {
+ describe("Time to stat the root path");
+ timeToStatPath(new Path("/"));
+ }
+
+ private void timeToStatPath(Path path) throws IOException {
+ describe("Timing getFileStatus(\"%s\")", path);
+ MetricDiff metadataRequests =
+ new MetricDiff(fs, Statistic.OBJECT_METADATA_REQUESTS);
+ MetricDiff listRequests =
+ new MetricDiff(fs, Statistic.OBJECT_LIST_REQUESTS);
+ long attempts = getOperationCount();
+ NanoTimer timer = new NanoTimer();
+ for (long l = 0; l < attempts; l++) {
+ fs.getFileStatus(path);
+ }
+ timer.end("Time to execute %d getFileStatusCalls", attempts);
+ LOG.info("Time per call: {}", toHuman(timer.nanosPerOperation(attempts)));
+ LOG.info("metadata: {}", metadataRequests);
+ LOG.info("metadata per operation {}", metadataRequests.diff() / attempts);
+ LOG.info("listObjects: {}", listRequests);
+ LOG.info("listObjects: per operation {}", listRequests.diff() / attempts);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
new file mode 100644
index 0000000..e2163c5
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
@@ -0,0 +1,534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.scale;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AInputPolicy;
+import org.apache.hadoop.fs.s3a.S3AInputStream;
+import org.apache.hadoop.fs.s3a.S3AInstrumentation;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.util.LineReader;
+import org.junit.After;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * Look at the performance of S3a operations.
+ */
+public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ITestS3AInputStreamPerformance.class);
+
+ private S3AFileSystem s3aFS;
+ private Path testData;
+ private S3AFileStatus testDataStatus;
+ private FSDataInputStream in;
+ private S3AInstrumentation.InputStreamStatistics streamStatistics;
+ public static final int BLOCK_SIZE = 32 * 1024;
+ public static final int BIG_BLOCK_SIZE = 256 * 1024;
+
+ /** Tests only run if the there is a named test file that can be read. */
+ private boolean testDataAvailable = true;
+ private String assumptionMessage = "test file";
+
+ /**
+ * Open the FS and the test data. The input stream is always set up here.
+ * @throws IOException IO Problems.
+ */
+ @Before
+ public void openFS() throws IOException {
+ Configuration conf = getConf();
+ conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024);
+ conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024);
+ String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
+ if (testFile.isEmpty()) {
+ assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;
+ testDataAvailable = false;
+ } else {
+ S3ATestUtils.useCSVDataEndpoint(conf);
+ testData = new Path(testFile);
+ Path path = this.testData;
+ bindS3aFS(path);
+ try {
+ testDataStatus = s3aFS.getFileStatus(this.testData);
+ } catch (IOException e) {
+ LOG.warn("Failed to read file {} specified in {}",
+ testFile, KEY_CSVTEST_FILE, e);
+ throw e;
+ }
+ }
+ }
+
+ private void bindS3aFS(Path path) throws IOException {
+ s3aFS = (S3AFileSystem) FileSystem.newInstance(path.toUri(), getConf());
+ }
+
+ /**
+ * Cleanup: close the stream, close the FS.
+ */
+ @After
+ public void cleanup() {
+ describe("cleanup");
+ IOUtils.closeStream(in);
+ IOUtils.closeStream(s3aFS);
+ }
+
+ /**
+ * Declare that the test requires the CSV test dataset.
+ */
+ private void requireCSVTestData() {
+ Assume.assumeTrue(assumptionMessage, testDataAvailable);
+ }
+
+ /**
+ * Open the test file with the read buffer specified in the setting.
+ * {@link #KEY_READ_BUFFER_SIZE}; use the {@code Normal} policy
+ * @return the stream, wrapping an S3a one
+ * @throws IOException IO problems
+ */
+ FSDataInputStream openTestFile() throws IOException {
+ return openTestFile(S3AInputPolicy.Normal, 0);
+ }
+
+ /**
+ * Open the test file with the read buffer specified in the setting
+ * {@link #KEY_READ_BUFFER_SIZE}.
+ * This includes the {@link #requireCSVTestData()} assumption; so
+ * if called before any FS op, will automatically skip the test
+ * if the CSV file is absent.
+ *
+ * @param inputPolicy input policy to use
+ * @param readahead readahead/buffer size
+ * @return the stream, wrapping an S3a one
+ * @throws IOException IO problems
+ */
+ FSDataInputStream openTestFile(S3AInputPolicy inputPolicy, long readahead)
+ throws IOException {
+ requireCSVTestData();
+ return openDataFile(s3aFS, this.testData, inputPolicy, readahead);
+ }
+
+ /**
+ * Open a test file with the read buffer specified in the setting
+ * {@link #KEY_READ_BUFFER_SIZE}.
+ *
+ * @param path path to open
+ * @param inputPolicy input policy to use
+ * @param readahead readahead/buffer size
+ * @return the stream, wrapping an S3a one
+ * @throws IOException IO problems
+ */
+ private FSDataInputStream openDataFile(S3AFileSystem fs,
+ Path path,
+ S3AInputPolicy inputPolicy,
+ long readahead) throws IOException {
+ int bufferSize = getConf().getInt(KEY_READ_BUFFER_SIZE,
+ DEFAULT_READ_BUFFER_SIZE);
+ S3AInputPolicy policy = fs.getInputPolicy();
+ fs.setInputPolicy(inputPolicy);
+ try {
+ FSDataInputStream stream = fs.open(path, bufferSize);
+ if (readahead >= 0) {
+ stream.setReadahead(readahead);
+ }
+ streamStatistics = getInputStreamStatistics(stream);
+ return stream;
+ } finally {
+ fs.setInputPolicy(policy);
+ }
+ }
+
+ /**
+ * Assert that the stream was only ever opened once.
+ */
+ protected void assertStreamOpenedExactlyOnce() {
+ assertOpenOperationCount(1);
+ }
+
+ /**
+ * Make an assertion count about the number of open operations.
+ * @param expected the expected number
+ */
+ private void assertOpenOperationCount(long expected) {
+ assertEquals("open operations in\n" + in,
+ expected, streamStatistics.openOperations);
+ }
+
+ /**
+ * Log how long an IOP took, by dividing the total time by the
+ * count of operations, printing in a human-readable form.
+ * @param operation operation being measured
+ * @param timer timing data
+ * @param count IOP count.
+ */
+ protected void logTimePerIOP(String operation,
+ NanoTimer timer,
+ long count) {
+ LOG.info("Time per {}: {} nS",
+ operation, toHuman(timer.duration() / count));
+ }
+
+ @Test
+ public void testTimeToOpenAndReadWholeFileBlocks() throws Throwable {
+ requireCSVTestData();
+ int blockSize = _1MB;
+ describe("Open the test file %s and read it in blocks of size %d",
+ testData, blockSize);
+ long len = testDataStatus.getLen();
+ in = openTestFile();
+ byte[] block = new byte[blockSize];
+ NanoTimer timer2 = new NanoTimer();
+ long count = 0;
+ // implicitly rounding down here
+ long blockCount = len / blockSize;
+ for (long i = 0; i < blockCount; i++) {
+ int offset = 0;
+ int remaining = blockSize;
+ NanoTimer blockTimer = new NanoTimer();
+ int reads = 0;
+ while (remaining > 0) {
+ int bytesRead = in.read(block, offset, remaining);
+ reads++;
+ if (bytesRead == 1) {
+ break;
+ }
+ remaining -= bytesRead;
+ offset += bytesRead;
+ count += bytesRead;
+ }
+ blockTimer.end("Reading block %d in %d reads", i, reads);
+ }
+ timer2.end("Time to read %d bytes in %d blocks", len, blockCount);
+ bandwidth(timer2, count);
+ logStreamStatistics();
+ }
+
+ @Test
+ public void testLazySeekEnabled() throws Throwable {
+ describe("Verify that seeks do not trigger any IO");
+ in = openTestFile();
+ long len = testDataStatus.getLen();
+ NanoTimer timer = new NanoTimer();
+ long blockCount = len / BLOCK_SIZE;
+ for (long i = 0; i < blockCount; i++) {
+ in.seek(in.getPos() + BLOCK_SIZE - 1);
+ }
+ in.seek(0);
+ blockCount++;
+ timer.end("Time to execute %d seeks", blockCount);
+ logTimePerIOP("seek()", timer, blockCount);
+ logStreamStatistics();
+ assertOpenOperationCount(0);
+ assertEquals("bytes read", 0, streamStatistics.bytesRead);
+ }
+
+ @Test
+ public void testReadaheadOutOfRange() throws Throwable {
+ try {
+ in = openTestFile();
+ in.setReadahead(-1L);
+ fail("Stream should have rejected the request "+ in);
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testReadWithNormalPolicy() throws Throwable {
+ describe("Read big blocks with a big readahead");
+ executeSeekReadSequence(BIG_BLOCK_SIZE, BIG_BLOCK_SIZE * 2,
+ S3AInputPolicy.Normal);
+ assertStreamOpenedExactlyOnce();
+ }
+
+ @Test
+ public void testDecompressionSequential128K() throws Throwable {
+ describe("Decompress with a 128K readahead");
+ executeDecompression(128 * 1024, S3AInputPolicy.Sequential);
+ assertStreamOpenedExactlyOnce();
+ }
+
+ /**
+ * Execute a decompression + line read with the given input policy.
+ * @param readahead byte readahead
+ * @param inputPolicy read policy
+ * @throws IOException IO Problems
+ */
+ private void executeDecompression(long readahead,
+ S3AInputPolicy inputPolicy) throws IOException {
+ CompressionCodecFactory factory
+ = new CompressionCodecFactory(getConf());
+ CompressionCodec codec = factory.getCodec(testData);
+ long bytesRead = 0;
+ int lines = 0;
+
+ FSDataInputStream objectIn = openTestFile(inputPolicy, readahead);
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ try (LineReader lineReader = new LineReader(
+ codec.createInputStream(objectIn), getConf())) {
+ Text line = new Text();
+ int read;
+ while ((read = lineReader.readLine(line)) > 0) {
+ bytesRead += read;
+ lines++;
+ }
+ } catch (EOFException eof) {
+ // done
+ }
+ timer.end("Time to read %d lines [%d bytes expanded, %d raw]" +
+ " with readahead = %d",
+ lines,
+ bytesRead,
+ testDataStatus.getLen(),
+ readahead);
+ logTimePerIOP("line read", timer, lines);
+ logStreamStatistics();
+ }
+
+ private void logStreamStatistics() {
+ LOG.info(String.format("Stream Statistics%n{}"), streamStatistics);
+ }
+
+ /**
+ * Execute a seek+read sequence.
+ * @param blockSize block size for seeks
+ * @param readahead what the readahead value of the stream should be
+ * @throws IOException IO problems
+ */
+ protected void executeSeekReadSequence(long blockSize,
+ long readahead,
+ S3AInputPolicy policy) throws IOException {
+ in = openTestFile(policy, readahead);
+ long len = testDataStatus.getLen();
+ NanoTimer timer = new NanoTimer();
+ long blockCount = len / blockSize;
+ LOG.info("Reading {} blocks, readahead = {}",
+ blockCount, readahead);
+ for (long i = 0; i < blockCount; i++) {
+ in.seek(in.getPos() + blockSize - 1);
+ // this is the read
+ assertTrue(in.read() >= 0);
+ }
+ timer.end("Time to execute %d seeks of distance %d with readahead = %d",
+ blockCount,
+ blockSize,
+ readahead);
+ logTimePerIOP("seek(pos + " + blockCount+"); read()", timer, blockCount);
+ LOG.info("Effective bandwidth {} MB/S",
+ timer.bandwidthDescription(streamStatistics.bytesRead -
+ streamStatistics.bytesSkippedOnSeek));
+ logStreamStatistics();
+ }
+
+ public static final int _4K = 4 * 1024;
+ public static final int _8K = 8 * 1024;
+ public static final int _16K = 16 * 1024;
+ public static final int _32K = 32 * 1024;
+ public static final int _64K = 64 * 1024;
+ public static final int _128K = 128 * 1024;
+ public static final int _256K = 256 * 1024;
+ public static final int _1MB = 1024 * 1024;
+ public static final int _2MB = 2 * _1MB;
+ public static final int _10MB = _1MB * 10;
+ public static final int _5MB = _1MB * 5;
+
+ private static final int[][] RANDOM_IO_SEQUENCE = {
+ {_2MB, _128K},
+ {_128K, _128K},
+ {_5MB, _64K},
+ {_1MB, _1MB},
+ };
+
+ @Test
+ public void testRandomIORandomPolicy() throws Throwable {
+ executeRandomIO(S3AInputPolicy.Random, (long) RANDOM_IO_SEQUENCE.length);
+ assertEquals("streams aborted in " + streamStatistics,
+ 0, streamStatistics.aborted);
+ }
+
+ @Test
+ public void testRandomIONormalPolicy() throws Throwable {
+ long expectedOpenCount = RANDOM_IO_SEQUENCE.length;
+ executeRandomIO(S3AInputPolicy.Normal, expectedOpenCount);
+ assertEquals("streams aborted in " + streamStatistics,
+ 4, streamStatistics.aborted);
+ }
+
+ /**
+ * Execute the random IO {@code readFully(pos, bytes[])} sequence defined by
+ * {@link #RANDOM_IO_SEQUENCE}. The stream is closed afterwards; that's used
+ * in the timing too
+ * @param policy read policy
+ * @param expectedOpenCount expected number of stream openings
+ * @throws IOException IO problems
+ * @return the timer
+ */
+ private ContractTestUtils.NanoTimer executeRandomIO(S3AInputPolicy policy,
+ long expectedOpenCount)
+ throws IOException {
+ describe("Random IO with policy \"%s\"", policy);
+ byte[] buffer = new byte[_1MB];
+ long totalBytesRead = 0;
+
+ in = openTestFile(policy, 0);
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ for (int[] action : RANDOM_IO_SEQUENCE) {
+ int position = action[0];
+ int range = action[1];
+ in.readFully(position, buffer, 0, range);
+ totalBytesRead += range;
+ }
+ int reads = RANDOM_IO_SEQUENCE.length;
+ timer.end("Time to execute %d reads of total size %d bytes",
+ reads,
+ totalBytesRead);
+ in.close();
+ assertOpenOperationCount(expectedOpenCount);
+ logTimePerIOP("byte read", timer, totalBytesRead);
+ LOG.info("Effective bandwidth {} MB/S",
+ timer.bandwidthDescription(streamStatistics.bytesRead -
+ streamStatistics.bytesSkippedOnSeek));
+ logStreamStatistics();
+ return timer;
+ }
+
+ S3AInputStream getS3aStream() {
+ return (S3AInputStream) in.getWrappedStream();
+ }
+
+ @Test
+ public void testRandomReadOverBuffer() throws Throwable {
+ describe("read over a buffer, making sure that the requests" +
+ " spans readahead ranges");
+ int datasetLen = _32K;
+ Path dataFile = new Path(getTestPath(), "testReadOverBuffer.bin");
+ byte[] sourceData = dataset(datasetLen, 0, 64);
+ // relies on the field 'fs' referring to the R/W FS
+ writeDataset(fs, dataFile, sourceData, datasetLen, _16K, true);
+ byte[] buffer = new byte[datasetLen];
+ int readahead = _8K;
+ int halfReadahead = _4K;
+ in = openDataFile(fs, dataFile, S3AInputPolicy.Random, readahead);
+
+ LOG.info("Starting initial reads");
+ S3AInputStream s3aStream = getS3aStream();
+ assertEquals(readahead, s3aStream.getReadahead());
+ byte[] oneByte = new byte[1];
+ assertEquals(1, in.read(0, oneByte, 0, 1));
+ // make some assertions about the current state
+ assertEquals("remaining in\n" + in,
+ readahead - 1, s3aStream.remainingInCurrentRequest());
+ assertEquals("range start in\n" + in,
+ 0, s3aStream.getContentRangeStart());
+ assertEquals("range finish in\n" + in,
+ readahead, s3aStream.getContentRangeFinish());
+
+ assertStreamOpenedExactlyOnce();
+
+ describe("Starting sequence of positioned read calls over\n%s", in);
+ NanoTimer readTimer = new NanoTimer();
+ int currentPos = halfReadahead;
+ int offset = currentPos;
+ int bytesRead = 0;
+ int readOps = 0;
+
+ // make multiple read() calls
+ while (bytesRead < halfReadahead) {
+ int length = buffer.length - offset;
+ int read = in.read(currentPos, buffer, offset, length);
+ bytesRead += read;
+ offset += read;
+ readOps++;
+ assertEquals("open operations on request #" + readOps
+ + " after reading " + bytesRead
+ + " current position in stream " + currentPos
+ + " in\n" + fs
+ + "\n " + in,
+ 1, streamStatistics.openOperations);
+ for (int i = currentPos; i < currentPos + read; i++) {
+ assertEquals("Wrong value from byte " + i,
+ sourceData[i], buffer[i]);
+ }
+ currentPos += read;
+ }
+ assertStreamOpenedExactlyOnce();
+ // assert at the end of the original block
+ assertEquals(readahead, currentPos);
+ readTimer.end("read %d in %d operations", bytesRead, readOps);
+ bandwidth(readTimer, bytesRead);
+ LOG.info("Time per byte(): {} nS",
+ toHuman(readTimer.nanosPerOperation(bytesRead)));
+ LOG.info("Time per read(): {} nS",
+ toHuman(readTimer.nanosPerOperation(readOps)));
+
+ describe("read last byte");
+ // read one more
+ int read = in.read(currentPos, buffer, bytesRead, 1);
+ assertTrue("-1 from last read", read >= 0);
+ assertOpenOperationCount(2);
+ assertEquals("Wrong value from read ", sourceData[currentPos],
+ (int) buffer[currentPos]);
+ currentPos++;
+
+
+ // now scan all the way to the end of the file, using single byte read()
+ // calls
+ describe("read() to EOF over \n%s", in);
+ long readCount = 0;
+ NanoTimer timer = new NanoTimer();
+ LOG.info("seeking");
+ in.seek(currentPos);
+ LOG.info("reading");
+ while(currentPos < datasetLen) {
+ int r = in.read();
+ assertTrue("Negative read() at position " + currentPos + " in\n" + in,
+ r >= 0);
+ buffer[currentPos] = (byte)r;
+ assertEquals("Wrong value from read from\n" + in,
+ sourceData[currentPos], r);
+ currentPos++;
+ readCount++;
+ }
+ timer.end("read %d bytes", readCount);
+ bandwidth(timer, readCount);
+ LOG.info("Time per read(): {} nS",
+ toHuman(timer.nanosPerOperation(readCount)));
+
+ assertEquals("last read in " + in, -1, in.read());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADeleteManyFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADeleteManyFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADeleteManyFiles.java
deleted file mode 100644
index 5e07dcb..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADeleteManyFiles.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.scale;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- * Test some scalable operations related to file renaming and deletion.
- */
-public class TestS3ADeleteManyFiles extends S3AScaleTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestS3ADeleteManyFiles.class);
-
- /**
- * CAUTION: If this test starts failing, please make sure that the
- * {@link org.apache.hadoop.fs.s3a.Constants#MAX_THREADS} configuration is not
- * set too low. Alternatively, consider reducing the
- * <code>scale.test.operation.count</code> parameter in
- * <code>getOperationCount()</code>.
- *
- * @see #getOperationCount()
- */
- @Test
- public void testBulkRenameAndDelete() throws Throwable {
- final Path scaleTestDir = getTestPath();
- final Path srcDir = new Path(scaleTestDir, "src");
- final Path finalDir = new Path(scaleTestDir, "final");
- final long count = getOperationCount();
- ContractTestUtils.rm(fs, scaleTestDir, true, false);
-
- fs.mkdirs(srcDir);
- fs.mkdirs(finalDir);
-
- int testBufferSize = fs.getConf()
- .getInt(ContractTestUtils.IO_CHUNK_BUFFER_SIZE,
- ContractTestUtils.DEFAULT_IO_CHUNK_BUFFER_SIZE);
- // use Executor to speed up file creation
- ExecutorService exec = Executors.newFixedThreadPool(16);
- final ExecutorCompletionService<Boolean> completionService =
- new ExecutorCompletionService<>(exec);
- try {
- final byte[] data = ContractTestUtils.dataset(testBufferSize, 'a', 'z');
-
- for (int i = 0; i < count; ++i) {
- final String fileName = "foo-" + i;
- completionService.submit(new Callable<Boolean>() {
- @Override
- public Boolean call() throws IOException {
- ContractTestUtils.createFile(fs, new Path(srcDir, fileName),
- false, data);
- return fs.exists(new Path(srcDir, fileName));
- }
- });
- }
- for (int i = 0; i < count; ++i) {
- final Future<Boolean> future = completionService.take();
- try {
- if (!future.get()) {
- LOG.warn("cannot create file");
- }
- } catch (ExecutionException e) {
- LOG.warn("Error while uploading file", e.getCause());
- throw e;
- }
- }
- } finally {
- exec.shutdown();
- }
-
- int nSrcFiles = fs.listStatus(srcDir).length;
- fs.rename(srcDir, finalDir);
- assertEquals(nSrcFiles, fs.listStatus(finalDir).length);
- ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
- new Path(srcDir, "foo-" + 0));
- ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
- new Path(srcDir, "foo-" + count / 2));
- ContractTestUtils.assertPathDoesNotExist(fs, "not deleted after rename",
- new Path(srcDir, "foo-" + (count - 1)));
- ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
- new Path(finalDir, "foo-" + 0));
- ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
- new Path(finalDir, "foo-" + count/2));
- ContractTestUtils.assertPathExists(fs, "not renamed to dest dir",
- new Path(finalDir, "foo-" + (count-1)));
-
- ContractTestUtils.assertDeleted(fs, finalDir, true, false);
- }
-
- @Test
- public void testOpenCreate() throws IOException {
- Path dir = new Path("/tests3a");
- ContractTestUtils.createAndVerifyFile(fs, dir, 1024);
- ContractTestUtils.createAndVerifyFile(fs, dir, 5 * 1024 * 1024);
- ContractTestUtils.createAndVerifyFile(fs, dir, 20 * 1024 * 1024);
-
-
- /*
- Enable to test the multipart upload
- try {
- ContractTestUtils.createAndVerifyFile(fs, dir,
- (long)6 * 1024 * 1024 * 1024);
- } catch (IOException e) {
- fail(e.getMessage());
- }
- */
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
deleted file mode 100644
index 35ea3ad..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.scale;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.Statistic;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.s3a.Statistic.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-
-/**
- * Test the performance of listing files/directories.
- */
-public class TestS3ADirectoryPerformance extends S3AScaleTestBase {
- private static final Logger LOG = LoggerFactory.getLogger(
- TestS3ADirectoryPerformance.class);
-
- @Test
- public void testListOperations() throws Throwable {
- describe("Test recursive list operations");
- final Path scaleTestDir = getTestPath();
- final Path listDir = new Path(scaleTestDir, "lists");
-
- // scale factor.
- int scale = getConf().getInt(KEY_DIRECTORY_COUNT, DEFAULT_DIRECTORY_COUNT);
- int width = scale;
- int depth = scale;
- int files = scale;
- MetricDiff metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
- MetricDiff listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
- MetricDiff listContinueRequests =
- new MetricDiff(fs, OBJECT_CONTINUE_LIST_REQUESTS);
- MetricDiff listStatusCalls = new MetricDiff(fs, INVOCATION_LIST_FILES);
- MetricDiff getFileStatusCalls =
- new MetricDiff(fs, INVOCATION_GET_FILE_STATUS);
- NanoTimer createTimer = new NanoTimer();
- TreeScanResults created =
- createSubdirs(fs, listDir, depth, width, files, 0);
- // add some empty directories
- int emptyDepth = 1 * scale;
- int emptyWidth = 3 * scale;
-
- created.add(createSubdirs(fs, listDir, emptyDepth, emptyWidth, 0,
- 0, "empty", "f-", ""));
- createTimer.end("Time to create %s", created);
- LOG.info("Time per operation: {}",
- toHuman(createTimer.nanosPerOperation(created.totalCount())));
- printThenReset(LOG,
- metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
-
- describe("Listing files via treewalk");
- try {
- // Scan the directory via an explicit tree walk.
- // This is the baseline for any listing speedups.
- NanoTimer treeWalkTimer = new NanoTimer();
- TreeScanResults treewalkResults = treeWalk(fs, listDir);
- treeWalkTimer.end("List status via treewalk of %s", created);
-
- printThenReset(LOG,
- metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
- assertEquals("Files found in listFiles(recursive=true) " +
- " created=" + created + " listed=" + treewalkResults,
- created.getFileCount(), treewalkResults.getFileCount());
-
- describe("Listing files via listFiles(recursive=true)");
- // listFiles() does the recursion internally
- NanoTimer listFilesRecursiveTimer = new NanoTimer();
-
- TreeScanResults listFilesResults = new TreeScanResults(
- fs.listFiles(listDir, true));
-
- listFilesRecursiveTimer.end("listFiles(recursive=true) of %s", created);
- assertEquals("Files found in listFiles(recursive=true) " +
- " created=" + created + " listed=" + listFilesResults,
- created.getFileCount(), listFilesResults.getFileCount());
-
- // only two list operations should have taken place
- print(LOG,
- metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
- assertEquals(listRequests.toString(), 2, listRequests.diff());
- reset(metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
-
-
- } finally {
- describe("deletion");
- // deletion at the end of the run
- NanoTimer deleteTimer = new NanoTimer();
- fs.delete(listDir, true);
- deleteTimer.end("Deleting directory tree");
- printThenReset(LOG,
- metadataRequests,
- listRequests,
- listContinueRequests,
- listStatusCalls,
- getFileStatusCalls);
- }
- }
-
- @Test
- public void testTimeToStatEmptyDirectory() throws Throwable {
- describe("Time to stat an empty directory");
- Path path = new Path(getTestPath(), "empty");
- fs.mkdirs(path);
- timeToStatPath(path);
- }
-
- @Test
- public void testTimeToStatNonEmptyDirectory() throws Throwable {
- describe("Time to stat a non-empty directory");
- Path path = new Path(getTestPath(), "dir");
- fs.mkdirs(path);
- touch(fs, new Path(path, "file"));
- timeToStatPath(path);
- }
-
- @Test
- public void testTimeToStatFile() throws Throwable {
- describe("Time to stat a simple file");
- Path path = new Path(getTestPath(), "file");
- touch(fs, path);
- timeToStatPath(path);
- }
-
- @Test
- public void testTimeToStatRoot() throws Throwable {
- describe("Time to stat the root path");
- timeToStatPath(new Path("/"));
- }
-
- private void timeToStatPath(Path path) throws IOException {
- describe("Timing getFileStatus(\"%s\")", path);
- MetricDiff metadataRequests =
- new MetricDiff(fs, Statistic.OBJECT_METADATA_REQUESTS);
- MetricDiff listRequests =
- new MetricDiff(fs, Statistic.OBJECT_LIST_REQUESTS);
- long attempts = getOperationCount();
- NanoTimer timer = new NanoTimer();
- for (long l = 0; l < attempts; l++) {
- fs.getFileStatus(path);
- }
- timer.end("Time to execute %d getFileStatusCalls", attempts);
- LOG.info("Time per call: {}", toHuman(timer.nanosPerOperation(attempts)));
- LOG.info("metadata: {}", metadataRequests);
- LOG.info("metadata per operation {}", metadataRequests.diff() / attempts);
- LOG.info("listObjects: {}", listRequests);
- LOG.info("listObjects: per operation {}", listRequests.diff() / attempts);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
deleted file mode 100644
index d6d9d66..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
+++ /dev/null
@@ -1,534 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.scale;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.s3a.S3AFileStatus;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3AInputPolicy;
-import org.apache.hadoop.fs.s3a.S3AInputStream;
-import org.apache.hadoop.fs.s3a.S3AInstrumentation;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.util.LineReader;
-import org.junit.After;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
-import static org.apache.hadoop.fs.s3a.Constants.*;
-
-/**
- * Look at the performance of S3a operations.
- */
-public class TestS3AInputStreamPerformance extends S3AScaleTestBase {
- private static final Logger LOG = LoggerFactory.getLogger(
- TestS3AInputStreamPerformance.class);
-
- private S3AFileSystem s3aFS;
- private Path testData;
- private S3AFileStatus testDataStatus;
- private FSDataInputStream in;
- private S3AInstrumentation.InputStreamStatistics streamStatistics;
- public static final int BLOCK_SIZE = 32 * 1024;
- public static final int BIG_BLOCK_SIZE = 256 * 1024;
-
- /** Tests only run if the there is a named test file that can be read. */
- private boolean testDataAvailable = true;
- private String assumptionMessage = "test file";
-
- /**
- * Open the FS and the test data. The input stream is always set up here.
- * @throws IOException IO Problems.
- */
- @Before
- public void openFS() throws IOException {
- Configuration conf = getConf();
- conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024);
- conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024);
- String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
- if (testFile.isEmpty()) {
- assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;
- testDataAvailable = false;
- } else {
- S3ATestUtils.useCSVDataEndpoint(conf);
- testData = new Path(testFile);
- Path path = this.testData;
- bindS3aFS(path);
- try {
- testDataStatus = s3aFS.getFileStatus(this.testData);
- } catch (IOException e) {
- LOG.warn("Failed to read file {} specified in {}",
- testFile, KEY_CSVTEST_FILE, e);
- throw e;
- }
- }
- }
-
- private void bindS3aFS(Path path) throws IOException {
- s3aFS = (S3AFileSystem) FileSystem.newInstance(path.toUri(), getConf());
- }
-
- /**
- * Cleanup: close the stream, close the FS.
- */
- @After
- public void cleanup() {
- describe("cleanup");
- IOUtils.closeStream(in);
- IOUtils.closeStream(s3aFS);
- }
-
- /**
- * Declare that the test requires the CSV test dataset.
- */
- private void requireCSVTestData() {
- Assume.assumeTrue(assumptionMessage, testDataAvailable);
- }
-
- /**
- * Open the test file with the read buffer specified in the setting.
- * {@link #KEY_READ_BUFFER_SIZE}; use the {@code Normal} policy
- * @return the stream, wrapping an S3a one
- * @throws IOException IO problems
- */
- FSDataInputStream openTestFile() throws IOException {
- return openTestFile(S3AInputPolicy.Normal, 0);
- }
-
- /**
- * Open the test file with the read buffer specified in the setting
- * {@link #KEY_READ_BUFFER_SIZE}.
- * This includes the {@link #requireCSVTestData()} assumption; so
- * if called before any FS op, will automatically skip the test
- * if the CSV file is absent.
- *
- * @param inputPolicy input policy to use
- * @param readahead readahead/buffer size
- * @return the stream, wrapping an S3a one
- * @throws IOException IO problems
- */
- FSDataInputStream openTestFile(S3AInputPolicy inputPolicy, long readahead)
- throws IOException {
- requireCSVTestData();
- return openDataFile(s3aFS, this.testData, inputPolicy, readahead);
- }
-
- /**
- * Open a test file with the read buffer specified in the setting
- * {@link #KEY_READ_BUFFER_SIZE}.
- *
- * @param path path to open
- * @param inputPolicy input policy to use
- * @param readahead readahead/buffer size
- * @return the stream, wrapping an S3a one
- * @throws IOException IO problems
- */
- private FSDataInputStream openDataFile(S3AFileSystem fs,
- Path path,
- S3AInputPolicy inputPolicy,
- long readahead) throws IOException {
- int bufferSize = getConf().getInt(KEY_READ_BUFFER_SIZE,
- DEFAULT_READ_BUFFER_SIZE);
- S3AInputPolicy policy = fs.getInputPolicy();
- fs.setInputPolicy(inputPolicy);
- try {
- FSDataInputStream stream = fs.open(path, bufferSize);
- if (readahead >= 0) {
- stream.setReadahead(readahead);
- }
- streamStatistics = getInputStreamStatistics(stream);
- return stream;
- } finally {
- fs.setInputPolicy(policy);
- }
- }
-
- /**
- * Assert that the stream was only ever opened once.
- */
- protected void assertStreamOpenedExactlyOnce() {
- assertOpenOperationCount(1);
- }
-
- /**
- * Make an assertion count about the number of open operations.
- * @param expected the expected number
- */
- private void assertOpenOperationCount(long expected) {
- assertEquals("open operations in\n" + in,
- expected, streamStatistics.openOperations);
- }
-
- /**
- * Log how long an IOP took, by dividing the total time by the
- * count of operations, printing in a human-readable form.
- * @param operation operation being measured
- * @param timer timing data
- * @param count IOP count.
- */
- protected void logTimePerIOP(String operation,
- NanoTimer timer,
- long count) {
- LOG.info("Time per {}: {} nS",
- operation, toHuman(timer.duration() / count));
- }
-
- @Test
- public void testTimeToOpenAndReadWholeFileBlocks() throws Throwable {
- requireCSVTestData();
- int blockSize = _1MB;
- describe("Open the test file %s and read it in blocks of size %d",
- testData, blockSize);
- long len = testDataStatus.getLen();
- in = openTestFile();
- byte[] block = new byte[blockSize];
- NanoTimer timer2 = new NanoTimer();
- long count = 0;
- // implicitly rounding down here
- long blockCount = len / blockSize;
- for (long i = 0; i < blockCount; i++) {
- int offset = 0;
- int remaining = blockSize;
- NanoTimer blockTimer = new NanoTimer();
- int reads = 0;
- while (remaining > 0) {
- int bytesRead = in.read(block, offset, remaining);
- reads ++;
- if (bytesRead == 1) {
- break;
- }
- remaining -= bytesRead;
- offset += bytesRead;
- count += bytesRead;
- }
- blockTimer.end("Reading block %d in %d reads", i, reads);
- }
- timer2.end("Time to read %d bytes in %d blocks", len, blockCount );
- bandwidth(timer2, count);
- logStreamStatistics();
- }
-
- @Test
- public void testLazySeekEnabled() throws Throwable {
- describe("Verify that seeks do not trigger any IO");
- in = openTestFile();
- long len = testDataStatus.getLen();
- NanoTimer timer = new NanoTimer();
- long blockCount = len / BLOCK_SIZE;
- for (long i = 0; i < blockCount; i++) {
- in.seek(in.getPos() + BLOCK_SIZE - 1);
- }
- in.seek(0);
- blockCount++;
- timer.end("Time to execute %d seeks", blockCount);
- logTimePerIOP("seek()", timer, blockCount);
- logStreamStatistics();
- assertOpenOperationCount(0);
- assertEquals("bytes read", 0, streamStatistics.bytesRead);
- }
-
- @Test
- public void testReadaheadOutOfRange() throws Throwable {
- try {
- in = openTestFile();
- in.setReadahead(-1L);
- fail("Stream should have rejected the request "+ in);
- } catch (IllegalArgumentException e) {
- // expected
- }
- }
-
- @Test
- public void testReadWithNormalPolicy() throws Throwable {
- describe("Read big blocks with a big readahead");
- executeSeekReadSequence(BIG_BLOCK_SIZE, BIG_BLOCK_SIZE * 2,
- S3AInputPolicy.Normal);
- assertStreamOpenedExactlyOnce();
- }
-
- @Test
- public void testDecompressionSequential128K() throws Throwable {
- describe("Decompress with a 128K readahead");
- executeDecompression(128 * 1024, S3AInputPolicy.Sequential);
- assertStreamOpenedExactlyOnce();
- }
-
- /**
- * Execute a decompression + line read with the given input policy.
- * @param readahead byte readahead
- * @param inputPolicy read policy
- * @throws IOException IO Problems
- */
- private void executeDecompression(long readahead,
- S3AInputPolicy inputPolicy) throws IOException {
- CompressionCodecFactory factory
- = new CompressionCodecFactory(getConf());
- CompressionCodec codec = factory.getCodec(testData);
- long bytesRead = 0;
- int lines = 0;
-
- FSDataInputStream objectIn = openTestFile(inputPolicy, readahead);
- ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
- try (LineReader lineReader = new LineReader(
- codec.createInputStream(objectIn), getConf())) {
- Text line = new Text();
- int read;
- while ((read = lineReader.readLine(line)) > 0) {
- bytesRead += read;
- lines++;
- }
- } catch (EOFException eof) {
- // done
- }
- timer.end("Time to read %d lines [%d bytes expanded, %d raw]" +
- " with readahead = %d",
- lines,
- bytesRead,
- testDataStatus.getLen(),
- readahead);
- logTimePerIOP("line read", timer, lines);
- logStreamStatistics();
- }
-
- private void logStreamStatistics() {
- LOG.info(String.format("Stream Statistics%n{}"), streamStatistics);
- }
-
- /**
- * Execute a seek+read sequence.
- * @param blockSize block size for seeks
- * @param readahead what the readahead value of the stream should be
- * @throws IOException IO problems
- */
- protected void executeSeekReadSequence(long blockSize,
- long readahead,
- S3AInputPolicy policy) throws IOException {
- in = openTestFile(policy, readahead);
- long len = testDataStatus.getLen();
- NanoTimer timer = new NanoTimer();
- long blockCount = len / blockSize;
- LOG.info("Reading {} blocks, readahead = {}",
- blockCount, readahead);
- for (long i = 0; i < blockCount; i++) {
- in.seek(in.getPos() + blockSize - 1);
- // this is the read
- assertTrue(in.read() >= 0);
- }
- timer.end("Time to execute %d seeks of distance %d with readahead = %d",
- blockCount,
- blockSize,
- readahead);
- logTimePerIOP("seek(pos + " + blockCount+"); read()", timer, blockCount);
- LOG.info("Effective bandwidth {} MB/S",
- timer.bandwidthDescription(streamStatistics.bytesRead -
- streamStatistics.bytesSkippedOnSeek));
- logStreamStatistics();
- }
-
- public static final int _4K = 4 * 1024;
- public static final int _8K = 8 * 1024;
- public static final int _16K = 16 * 1024;
- public static final int _32K = 32 * 1024;
- public static final int _64K = 64 * 1024;
- public static final int _128K = 128 * 1024;
- public static final int _256K = 256 * 1024;
- public static final int _1MB = 1024 * 1024;
- public static final int _2MB = 2 * _1MB;
- public static final int _10MB = _1MB * 10;
- public static final int _5MB = _1MB * 5;
-
- private static final int[][] RANDOM_IO_SEQUENCE = {
- {_2MB, _128K},
- {_128K, _128K},
- {_5MB, _64K},
- {_1MB, _1MB},
- };
-
- @Test
- public void testRandomIORandomPolicy() throws Throwable {
- executeRandomIO(S3AInputPolicy.Random, (long) RANDOM_IO_SEQUENCE.length);
- assertEquals("streams aborted in " + streamStatistics,
- 0, streamStatistics.aborted);
- }
-
- @Test
- public void testRandomIONormalPolicy() throws Throwable {
- long expectedOpenCount = RANDOM_IO_SEQUENCE.length;
- executeRandomIO(S3AInputPolicy.Normal, expectedOpenCount);
- assertEquals("streams aborted in " + streamStatistics,
- 4, streamStatistics.aborted);
- }
-
- /**
- * Execute the random IO {@code readFully(pos, bytes[])} sequence defined by
- * {@link #RANDOM_IO_SEQUENCE}. The stream is closed afterwards; that's used
- * in the timing too
- * @param policy read policy
- * @param expectedOpenCount expected number of stream openings
- * @throws IOException IO problems
- * @return the timer
- */
- private ContractTestUtils.NanoTimer executeRandomIO(S3AInputPolicy policy,
- long expectedOpenCount)
- throws IOException {
- describe("Random IO with policy \"%s\"", policy);
- byte[] buffer = new byte[_1MB];
- long totalBytesRead = 0;
-
- in = openTestFile(policy, 0);
- ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
- for (int[] action : RANDOM_IO_SEQUENCE) {
- int position = action[0];
- int range = action[1];
- in.readFully(position, buffer, 0, range);
- totalBytesRead += range;
- }
- int reads = RANDOM_IO_SEQUENCE.length;
- timer.end("Time to execute %d reads of total size %d bytes",
- reads,
- totalBytesRead);
- in.close();
- assertOpenOperationCount(expectedOpenCount);
- logTimePerIOP("byte read", timer, totalBytesRead);
- LOG.info("Effective bandwidth {} MB/S",
- timer.bandwidthDescription(streamStatistics.bytesRead -
- streamStatistics.bytesSkippedOnSeek));
- logStreamStatistics();
- return timer;
- }
-
- S3AInputStream getS3aStream() {
- return (S3AInputStream) in.getWrappedStream();
- }
-
- @Test
- public void testRandomReadOverBuffer() throws Throwable {
- describe("read over a buffer, making sure that the requests" +
- " spans readahead ranges");
- int datasetLen = _32K;
- Path dataFile = new Path(getTestPath(), "testReadOverBuffer.bin");
- byte[] sourceData = dataset(datasetLen, 0, 64);
- // relies on the field 'fs' referring to the R/W FS
- writeDataset(fs, dataFile, sourceData, datasetLen, _16K, true);
- byte[] buffer = new byte[datasetLen];
- int readahead = _8K;
- int halfReadahead = _4K;
- in = openDataFile(fs, dataFile, S3AInputPolicy.Random, readahead);
-
- LOG.info("Starting initial reads");
- S3AInputStream s3aStream = getS3aStream();
- assertEquals(readahead, s3aStream.getReadahead());
- byte[] oneByte = new byte[1];
- assertEquals(1, in.read(0, oneByte, 0, 1));
- // make some assertions about the current state
- assertEquals("remaining in\n" + in,
- readahead - 1, s3aStream.remainingInCurrentRequest());
- assertEquals("range start in\n" + in,
- 0, s3aStream.getContentRangeStart());
- assertEquals("range finish in\n" + in,
- readahead, s3aStream.getContentRangeFinish());
-
- assertStreamOpenedExactlyOnce();
-
- describe("Starting sequence of positioned read calls over\n%s", in);
- NanoTimer readTimer = new NanoTimer();
- int currentPos = halfReadahead;
- int offset = currentPos;
- int bytesRead = 0;
- int readOps = 0;
-
- // make multiple read() calls
- while (bytesRead < halfReadahead) {
- int length = buffer.length - offset;
- int read = in.read(currentPos, buffer, offset, length);
- bytesRead += read;
- offset += read;
- readOps++;
- assertEquals("open operations on request #" + readOps
- + " after reading " + bytesRead
- + " current position in stream " + currentPos
- + " in\n" + fs
- + "\n " + in,
- 1, streamStatistics.openOperations);
- for (int i = currentPos; i < currentPos + read; i++) {
- assertEquals("Wrong value from byte " + i,
- sourceData[i], buffer[i]);
- }
- currentPos += read;
- }
- assertStreamOpenedExactlyOnce();
- // assert at the end of the original block
- assertEquals(readahead, currentPos);
- readTimer.end("read %d in %d operations", bytesRead, readOps);
- bandwidth(readTimer, bytesRead);
- LOG.info("Time per byte(): {} nS",
- toHuman(readTimer.nanosPerOperation(bytesRead)));
- LOG.info("Time per read(): {} nS",
- toHuman(readTimer.nanosPerOperation(readOps)));
-
- describe("read last byte");
- // read one more
- int read = in.read(currentPos, buffer, bytesRead, 1);
- assertTrue("-1 from last read", read >= 0);
- assertOpenOperationCount(2);
- assertEquals("Wrong value from read ", sourceData[currentPos],
- (int) buffer[currentPos]);
- currentPos++;
-
-
- // now scan all the way to the end of the file, using single byte read()
- // calls
- describe("read() to EOF over \n%s", in);
- long readCount = 0;
- NanoTimer timer = new NanoTimer();
- LOG.info("seeking");
- in.seek(currentPos);
- LOG.info("reading");
- while(currentPos < datasetLen) {
- int r = in.read();
- assertTrue("Negative read() at position " + currentPos + " in\n" + in,
- r >= 0);
- buffer[currentPos] = (byte)r;
- assertEquals("Wrong value from read from\n" + in,
- sourceData[currentPos], r);
- currentPos++;
- readCount++;
- }
- timer.end("read %d bytes", readCount);
- bandwidth(timer, readCount);
- LOG.info("Time per read(): {} nS",
- toHuman(timer.nanosPerOperation(readCount)));
-
- assertEquals("last read in " + in, -1, in.read());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java
new file mode 100644
index 0000000..ca57da6
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.s3a.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FsStatus;
+import org.apache.hadoop.fs.Path;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.util.EnumSet;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * S3A tests through the {@link FileContext} API.
+ */
+public class ITestS3A {
+ private FileContext fc;
+
+ @Rule
+ public final Timeout testTimeout = new Timeout(90000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fc = S3ATestUtils.createTestFileContext(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fc != null) {
+ fc.delete(getTestPath(), true);
+ }
+ }
+
+ protected Path getTestPath() {
+ return new Path("/tests3afc");
+ }
+
+ @Test
+ public void testS3AStatus() throws Exception {
+ FsStatus fsStatus = fc.getFsStatus(null);
+ assertNotNull(fsStatus);
+ assertTrue("Used capacity should be positive: " + fsStatus.getUsed(),
+ fsStatus.getUsed() >= 0);
+ assertTrue("Remaining capacity should be positive: " + fsStatus
+ .getRemaining(),
+ fsStatus.getRemaining() >= 0);
+ assertTrue("Capacity should be positive: " + fsStatus.getCapacity(),
+ fsStatus.getCapacity() >= 0);
+ }
+
+ @Test
+ public void testS3ACreateFileInSubDir() throws Exception {
+ Path dirPath = getTestPath();
+ fc.mkdir(dirPath, FileContext.DIR_DEFAULT_PERM, true);
+ Path filePath = new Path(dirPath, "file");
+ try (FSDataOutputStream file = fc.create(filePath, EnumSet.of(CreateFlag
+ .CREATE))) {
+ file.write(666);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
new file mode 100644
index 0000000..772d8c7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.fs.s3a.yarn;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.WordCount;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+
+import org.junit.After;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests that S3A is usable through a YARN application.
+ */
+public class ITestS3AMiniYarnCluster {
+
+ private final Configuration conf = new YarnConfiguration();
+ private S3AFileSystem fs;
+ private MiniYARNCluster yarnCluster;
+ private final String rootPath = "/tests/MiniClusterWordCount/";
+
+ @Before
+ public void beforeTest() throws IOException {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fs.mkdirs(new Path(rootPath + "input/"));
+
+ yarnCluster = new MiniYARNCluster("MiniClusterWordCount", // testName
+ 1, // number of node managers
+ 1, // number of local log dirs per node manager
+ 1); // number of hdfs dirs per node manager
+ yarnCluster.init(conf);
+ yarnCluster.start();
+ }
+
+ @After
+ public void afterTest() throws IOException {
+ fs.delete(new Path(rootPath), true);
+ yarnCluster.stop();
+ }
+
+ @Test
+ public void testWithMiniCluster() throws Exception {
+ Path input = new Path(rootPath + "input/in.txt");
+ input = input.makeQualified(fs.getUri(), fs.getWorkingDirectory());
+ Path output = new Path(rootPath + "output/");
+ output = output.makeQualified(fs.getUri(), fs.getWorkingDirectory());
+
+ writeStringToFile(input, "first line\nsecond line\nthird line");
+
+ Job job = Job.getInstance(conf, "word count");
+ job.setJarByClass(WordCount.class);
+ job.setMapperClass(WordCount.TokenizerMapper.class);
+ job.setCombinerClass(WordCount.IntSumReducer.class);
+ job.setReducerClass(WordCount.IntSumReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+ FileInputFormat.addInputPath(job, input);
+ FileOutputFormat.setOutputPath(job, output);
+
+ int exitCode = (job.waitForCompletion(true) ? 0 : 1);
+ assertEquals("Returned error code.", 0, exitCode);
+
+ assertTrue(fs.exists(new Path(output, "_SUCCESS")));
+ String outputAsStr = readStringFromFile(new Path(output, "part-r-00000"));
+ Map<String, Integer> resAsMap = getResultAsMap(outputAsStr);
+
+ assertEquals(4, resAsMap.size());
+ assertEquals(1, (int) resAsMap.get("first"));
+ assertEquals(1, (int) resAsMap.get("second"));
+ assertEquals(1, (int) resAsMap.get("third"));
+ assertEquals(3, (int) resAsMap.get("line"));
+ }
+
+ /**
+ * helper method.
+ */
+ private Map<String, Integer> getResultAsMap(String outputAsStr)
+ throws IOException {
+ Map<String, Integer> result = new HashMap<>();
+ for (String line : outputAsStr.split("\n")) {
+ String[] tokens = line.split("\t");
+ result.put(tokens[0], Integer.parseInt(tokens[1]));
+ }
+ return result;
+ }
+
+ /**
+ * helper method.
+ */
+ private void writeStringToFile(Path path, String string) throws IOException {
+ FileContext fc = S3ATestUtils.createTestFileContext(conf);
+ try (FSDataOutputStream file = fc.create(path,
+ EnumSet.of(CreateFlag.CREATE))) {
+ file.write(string.getBytes());
+ }
+ }
+
+ /**
+ * helper method.
+ */
+ private String readStringFromFile(Path path) {
+ try (FSDataInputStream in = fs.open(path)) {
+ long bytesLen = fs.getFileStatus(path).getLen();
+ byte[] buffer = new byte[(int) bytesLen];
+ IOUtils.readFully(in, buffer, 0, buffer.length);
+ return new String(buffer);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to read from [" + path + "]", e);
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/52] [abbrv] hadoop git commit: HDFS-10764. Fix
INodeFile#getBlocks to not return null. Contributed by Arpit Agarwal.
Posted by aw...@apache.org.
HDFS-10764. Fix INodeFile#getBlocks to not return null. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0faee62a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0faee62a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0faee62a
Branch: refs/heads/HADOOP-13341
Commit: 0faee62a0c8c1b8fd83227babfd00fbc2b26bddf
Parents: 99603e9
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Aug 19 22:13:36 2016 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Aug 19 22:13:36 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/INodeFile.java | 29 ++++++++------------
1 file changed, 11 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0faee62a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 63945a4..12ead7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -200,8 +200,8 @@ public class INodeFile extends INodeWithAdditionalFields
public INodeFile(INodeFile that) {
super(that);
this.header = that.header;
- this.blocks = that.blocks;
this.features = that.features;
+ setBlocks(that.blocks);
}
public INodeFile(INodeFile that, FileDiffList diffs) {
@@ -271,9 +271,6 @@ public class INodeFile extends INodeWithAdditionalFields
/** Assert all blocks are complete. */
private void assertAllBlocksComplete(int numCommittedAllowed,
short minReplication) {
- if (blocks == null) {
- return;
- }
for (int i = 0; i < blocks.length; i++) {
final String err = checkBlockComplete(blocks, i, numCommittedAllowed,
minReplication);
@@ -342,7 +339,7 @@ public class INodeFile extends INodeWithAdditionalFields
BlockInfo removeLastBlock(Block oldblock) {
Preconditions.checkState(isUnderConstruction(),
"file is no longer under construction");
- if (blocks == null || blocks.length == 0) {
+ if (blocks.length == 0) {
return null;
}
int size_1 = blocks.length - 1;
@@ -618,7 +615,7 @@ public class INodeFile extends INodeWithAdditionalFields
*/
void addBlock(BlockInfo newblock) {
Preconditions.checkArgument(newblock.isStriped() == this.isStriped());
- if (this.blocks == null) {
+ if (this.blocks.length == 0) {
this.setBlocks(new BlockInfo[]{newblock});
} else {
int size = this.blocks.length;
@@ -631,12 +628,12 @@ public class INodeFile extends INodeWithAdditionalFields
/** Set the blocks. */
private void setBlocks(BlockInfo[] blocks) {
- this.blocks = blocks;
+ this.blocks = (blocks != null ? blocks : BlockInfo.EMPTY_ARRAY);
}
/** Clear all blocks of the file. */
public void clearBlocks() {
- setBlocks(BlockInfo.EMPTY_ARRAY);
+ this.blocks = BlockInfo.EMPTY_ARRAY;
}
@Override
@@ -836,7 +833,7 @@ public class INodeFile extends INodeWithAdditionalFields
*/
public final long computeFileSize(boolean includesLastUcBlock,
boolean usePreferredBlockSize4LastUcBlock) {
- if (blocks == null || blocks.length == 0) {
+ if (blocks.length == 0) {
return 0;
}
final int last = blocks.length - 1;
@@ -876,10 +873,6 @@ public class INodeFile extends INodeWithAdditionalFields
// TODO: support EC with heterogeneous storage
public final QuotaCounts storagespaceConsumedStriped() {
QuotaCounts counts = new QuotaCounts.Builder().build();
- if (blocks == null || blocks.length == 0) {
- return counts;
- }
-
for (BlockInfo b : blocks) {
Preconditions.checkState(b.isStriped());
long blockSize = b.isComplete() ?
@@ -931,7 +924,7 @@ public class INodeFile extends INodeWithAdditionalFields
* Return the penultimate allocated block for this file.
*/
BlockInfo getPenultimateBlock() {
- if (blocks == null || blocks.length <= 1) {
+ if (blocks.length <= 1) {
return null;
}
return blocks[blocks.length - 2];
@@ -939,12 +932,12 @@ public class INodeFile extends INodeWithAdditionalFields
@Override
public BlockInfo getLastBlock() {
- return blocks == null || blocks.length == 0? null: blocks[blocks.length-1];
+ return blocks.length == 0 ? null: blocks[blocks.length-1];
}
@Override
public int numBlocks() {
- return blocks == null ? 0 : blocks.length;
+ return blocks.length;
}
@VisibleForTesting
@@ -955,7 +948,7 @@ public class INodeFile extends INodeWithAdditionalFields
out.print(", fileSize=" + computeFileSize(snapshotId));
// only compare the first block
out.print(", blocks=");
- out.print(blocks == null || blocks.length == 0? null: blocks[0]);
+ out.print(blocks.length == 0 ? null: blocks[0]);
out.println();
}
@@ -1004,7 +997,7 @@ public class INodeFile extends INodeWithAdditionalFields
long newLength, BlockStoragePolicy bsps,
QuotaCounts delta) {
final BlockInfo[] blocks = getBlocks();
- if (blocks == null || blocks.length == 0) {
+ if (blocks.length == 0) {
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/52] [abbrv] hadoop git commit: HADOOP-13487. Hadoop KMS should
load old delegation tokens from Zookeeper on startup. Contributed by Xiao
Chen.
Posted by aw...@apache.org.
HADOOP-13487. Hadoop KMS should load old delegation tokens from Zookeeper on startup. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4d4d347
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4d4d347
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4d4d347
Branch: refs/heads/HADOOP-13341
Commit: f4d4d3474cfd2d1f2d243f5ae5cec17af38270b1
Parents: 22fc46d
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Aug 22 14:31:13 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Aug 22 14:42:13 2016 -0700
----------------------------------------------------------------------
.../ZKDelegationTokenSecretManager.java | 44 +++++++++
.../TestZKDelegationTokenSecretManager.java | 93 +++++++++++++++++++-
2 files changed, 136 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d4d347/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
index c3ad9f3..6c66e98 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
@@ -361,6 +361,7 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
}
}
}, listenerThreadPool);
+ loadFromZKCache(false);
}
} catch (Exception e) {
throw new IOException("Could not start PathChildrenCache for keys", e);
@@ -389,6 +390,7 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
}
}
}, listenerThreadPool);
+ loadFromZKCache(true);
}
} catch (Exception e) {
throw new IOException("Could not start PathChildrenCache for tokens", e);
@@ -396,6 +398,43 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
super.startThreads();
}
+ /**
+ * Load the PathChildrenCache into the in-memory map. Possible caches to be
+ * loaded are keyCache and tokenCache.
+ *
+ * @param isTokenCache true if loading tokenCache, false if loading keyCache.
+ */
+ private void loadFromZKCache(final boolean isTokenCache) {
+ final String cacheName = isTokenCache ? "token" : "key";
+ LOG.info("Starting to load {} cache.", cacheName);
+ final List<ChildData> children;
+ if (isTokenCache) {
+ children = tokenCache.getCurrentData();
+ } else {
+ children = keyCache.getCurrentData();
+ }
+
+ int count = 0;
+ for (ChildData child : children) {
+ try {
+ if (isTokenCache) {
+ processTokenAddOrUpdate(child);
+ } else {
+ processKeyAddOrUpdate(child.getData());
+ }
+ } catch (Exception e) {
+ LOG.info("Ignoring node {} because it failed to load.",
+ child.getPath());
+ LOG.debug("Failure exception:", e);
+ ++count;
+ }
+ }
+ if (count > 0) {
+ LOG.warn("Ignored {} nodes while loading {} cache.", count, cacheName);
+ }
+ LOG.info("Loaded {} cache.", cacheName);
+ }
+
private void processKeyAddOrUpdate(byte[] data) throws IOException {
ByteArrayInputStream bin = new ByteArrayInputStream(data);
DataInputStream din = new DataInputStream(bin);
@@ -890,4 +929,9 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
public ExecutorService getListenerThreadPool() {
return listenerThreadPool;
}
+
+ @VisibleForTesting
+ DelegationTokenInformation getTokenInfoFromMemory(TokenIdent ident) {
+ return currentTokens.get(ident);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d4d347/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
index 185a994..c9571ff2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
+import com.google.common.base.Supplier;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -37,6 +38,7 @@ import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
@@ -44,12 +46,18 @@ import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.junit.Assert.fail;
-import org.junit.Test;
public class TestZKDelegationTokenSecretManager {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestZKDelegationTokenSecretManager.class);
private static final int TEST_RETRIES = 2;
@@ -61,6 +69,9 @@ public class TestZKDelegationTokenSecretManager {
private TestingServer zkServer;
+ @Rule
+ public Timeout globalTimeout = new Timeout(300000);
+
@Before
public void setup() throws Exception {
zkServer = new TestingServer();
@@ -382,4 +393,84 @@ public class TestZKDelegationTokenSecretManager {
}
}
+ @SuppressWarnings({ "unchecked" })
+ @Test
+ public void testNodesLoadedAfterRestart() throws Exception {
+ final String connectString = zkServer.getConnectString();
+ final Configuration conf = getSecretConf(connectString);
+ final int removeScan = 1;
+ // Set the remove scan interval to remove expired tokens
+ conf.setLong(DelegationTokenManager.REMOVAL_SCAN_INTERVAL, removeScan);
+ // Set the update interval to trigger background thread to run. The thread
+ // is hard-coded to sleep at least 5 seconds.
+ conf.setLong(DelegationTokenManager.UPDATE_INTERVAL, 5);
+ // Set token expire time to 5 seconds.
+ conf.setLong(DelegationTokenManager.RENEW_INTERVAL, 5);
+
+ DelegationTokenManager tm =
+ new DelegationTokenManager(conf, new Text("bla"));
+ tm.init();
+ Token<DelegationTokenIdentifier> token =
+ (Token<DelegationTokenIdentifier>) tm
+ .createToken(UserGroupInformation.getCurrentUser(), "good");
+ Assert.assertNotNull(token);
+ Token<DelegationTokenIdentifier> cancelled =
+ (Token<DelegationTokenIdentifier>) tm
+ .createToken(UserGroupInformation.getCurrentUser(), "cancelled");
+ Assert.assertNotNull(cancelled);
+ tm.verifyToken(token);
+ tm.verifyToken(cancelled);
+
+ // Cancel one token, verify it's gone
+ tm.cancelToken(cancelled, "cancelled");
+ final AbstractDelegationTokenSecretManager sm =
+ tm.getDelegationTokenSecretManager();
+ final ZKDelegationTokenSecretManager zksm =
+ (ZKDelegationTokenSecretManager) sm;
+ final AbstractDelegationTokenIdentifier idCancelled =
+ sm.decodeTokenIdentifier(cancelled);
+ LOG.info("Waiting for the cancelled token to be removed");
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ AbstractDelegationTokenSecretManager.DelegationTokenInformation dtinfo =
+ zksm.getTokenInfo(idCancelled);
+ return dtinfo == null;
+ }
+ }, 100, 5000);
+
+ // Fake a restart which launches a new tm
+ tm.destroy();
+ tm = new DelegationTokenManager(conf, new Text("bla"));
+ tm.init();
+ final AbstractDelegationTokenSecretManager smNew =
+ tm.getDelegationTokenSecretManager();
+ final ZKDelegationTokenSecretManager zksmNew =
+ (ZKDelegationTokenSecretManager) smNew;
+
+ // The cancelled token should be gone, and not loaded.
+ AbstractDelegationTokenIdentifier id =
+ smNew.decodeTokenIdentifier(cancelled);
+ AbstractDelegationTokenSecretManager.DelegationTokenInformation dtinfo =
+ zksmNew.getTokenInfo(id);
+ Assert.assertNull("canceled dt should be gone!", dtinfo);
+
+ // The good token should be loaded on startup, and removed after expiry.
+ id = smNew.decodeTokenIdentifier(token);
+ dtinfo = zksmNew.getTokenInfoFromMemory(id);
+ Assert.assertNotNull("good dt should be in memory!", dtinfo);
+
+ // Wait for the good token to expire.
+ Thread.sleep(5000);
+ final ZKDelegationTokenSecretManager zksm1 = zksmNew;
+ final AbstractDelegationTokenIdentifier id1 = id;
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ LOG.info("Waiting for the expired token to be removed...");
+ return zksm1.getTokenInfo(id1) == null;
+ }
+ }, 1000, 5000);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/52] [abbrv] hadoop git commit: MAPREDUCE-6587. Remove unused
params in connection-related methods of Fetcher. Contributed by Yiqun Lin.
Posted by aw...@apache.org.
MAPREDUCE-6587. Remove unused params in connection-related methods of Fetcher. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cc4a670
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cc4a670
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cc4a670
Branch: refs/heads/HADOOP-13341
Commit: 8cc4a67059e37b2083cd5468b35a64a403a3e3ae
Parents: c49333b
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Aug 23 17:04:55 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Aug 23 17:04:55 2016 +0900
----------------------------------------------------------------------
.../org/apache/hadoop/mapreduce/task/reduce/Fetcher.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc4a670/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index d8dd7b5..be2f84f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -263,7 +263,7 @@ class Fetcher<K,V> extends Thread {
DataInputStream input = null;
try {
- setupConnectionsWithRetry(host, remaining, url);
+ setupConnectionsWithRetry(url);
if (stopped) {
abortConnect(host, remaining);
} else {
@@ -374,9 +374,8 @@ class Fetcher<K,V> extends Thread {
}
}
- private void setupConnectionsWithRetry(MapHost host,
- Set<TaskAttemptID> remaining, URL url) throws IOException {
- openConnectionWithRetry(host, remaining, url);
+ private void setupConnectionsWithRetry(URL url) throws IOException {
+ openConnectionWithRetry(url);
if (stopped) {
return;
}
@@ -396,8 +395,7 @@ class Fetcher<K,V> extends Thread {
verifyConnection(url, msgToEncode, encHash);
}
- private void openConnectionWithRetry(MapHost host,
- Set<TaskAttemptID> remaining, URL url) throws IOException {
+ private void openConnectionWithRetry(URL url) throws IOException {
long startTime = Time.monotonicNow();
boolean shouldWait = true;
while (shouldWait) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/52] [abbrv] hadoop git commit: HDFS-10762. Pass IIP for file
status related methods. Contributed by Daryn Sharp.
Posted by aw...@apache.org.
HDFS-10762. Pass IIP for file status related methods. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22fc46d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22fc46d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22fc46d7
Branch: refs/heads/HADOOP-13341
Commit: 22fc46d7659972ff016ccf1c6f781f0c160be26f
Parents: dc7a1c5
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Aug 22 15:37:02 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Aug 22 15:37:02 2016 -0500
----------------------------------------------------------------------
.../hdfs/server/namenode/FSDirAppendOp.java | 6 +-
.../server/namenode/FSDirStatAndListingOp.java | 80 +++++++++-----------
.../hdfs/server/namenode/FSDirWriteFileOp.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 14 ++--
.../hdfs/server/namenode/INodesInPath.java | 42 ++++++++--
.../hadoop/hdfs/TestReservedRawPaths.java | 21 +++++
6 files changed, 102 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 3a5d7dc..5192352 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -85,9 +85,10 @@ final class FSDirAppendOp {
final LocatedBlock lb;
final FSDirectory fsd = fsn.getFSDirectory();
final String src;
+ final INodesInPath iip;
fsd.writeLock();
try {
- final INodesInPath iip = fsd.resolvePathForWrite(pc, srcArg);
+ iip = fsd.resolvePathForWrite(pc, srcArg);
src = iip.getPath();
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
@@ -148,8 +149,7 @@ final class FSDirAppendOp {
fsd.writeUnlock();
}
- HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, src, false,
- FSDirectory.isReservedRawName(srcArg));
+ HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, iip);
if (lb != null) {
NameNode.stateChangeLog.debug(
"DIR* NameSystem.appendFile: file {} for {} at {} block {} block"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index c9eedf5..88be510 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -108,16 +108,16 @@ class FSDirStatAndListingOp {
if (!DFSUtil.isValidName(src)) {
throw new InvalidPathException("Invalid file name: " + src);
}
+ final INodesInPath iip;
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- final INodesInPath iip = fsd.resolvePath(pc, srcArg, resolveLink);
- src = iip.getPath();
+ iip = fsd.resolvePath(pc, srcArg, resolveLink);
fsd.checkPermission(pc, iip, false, null, null, null, null, false);
} else {
src = FSDirectory.resolvePath(srcArg, fsd);
+ iip = fsd.getINodesInPath(src, resolveLink);
}
- return getFileInfo(fsd, src, FSDirectory.isReservedRawName(srcArg),
- resolveLink);
+ return getFileInfo(fsd, iip);
}
/**
@@ -230,7 +230,6 @@ class FSDirStatAndListingOp {
String src, byte[] startAfter, boolean needLocation, boolean isSuperUser)
throws IOException {
String srcs = FSDirectory.normalizePath(src);
- final boolean isRawPath = FSDirectory.isReservedRawName(src);
if (FSDirectory.isExactReservedName(srcs)) {
return getReservedListing(fsd);
}
@@ -257,7 +256,7 @@ class FSDirStatAndListingOp {
return new DirectoryListing(
new HdfsFileStatus[]{ createFileStatus(
fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- needLocation, parentStoragePolicy, snapshot, isRawPath, iip)
+ needLocation, parentStoragePolicy, iip)
}, 0);
}
@@ -282,7 +281,7 @@ class FSDirStatAndListingOp {
cur.getLocalNameBytes());
listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(), nodeAttrs,
needLocation, getStoragePolicyID(curPolicy, parentStoragePolicy),
- snapshot, isRawPath, iipWithChild);
+ iipWithChild);
listingCnt++;
if (needLocation) {
// Once we hit lsLimit locations, stop.
@@ -339,7 +338,6 @@ class FSDirStatAndListingOp {
listing[i] = createFileStatus(
fsd, sRoot.getLocalNameBytes(), nodeAttrs,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
- Snapshot.CURRENT_STATE_ID, false,
INodesInPath.fromINode(sRoot));
}
return new DirectoryListing(
@@ -363,10 +361,8 @@ class FSDirStatAndListingOp {
* @return object containing information regarding the file
* or null if file not found
*/
- static HdfsFileStatus getFileInfo(
- FSDirectory fsd, String path, INodesInPath iip, boolean isRawPath,
- boolean includeStoragePolicy)
- throws IOException {
+ static HdfsFileStatus getFileInfo(FSDirectory fsd,
+ INodesInPath iip, boolean includeStoragePolicy) throws IOException {
fsd.readLock();
try {
final INode node = iip.getLastINode();
@@ -377,36 +373,32 @@ class FSDirStatAndListingOp {
byte policyId = includeStoragePolicy && !node.isSymlink() ?
node.getStoragePolicyID() :
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
- INodeAttributes nodeAttrs = getINodeAttributes(fsd, path,
+ INodeAttributes nodeAttrs = getINodeAttributes(fsd, iip.getPath(),
HdfsFileStatus.EMPTY_NAME,
node, iip.getPathSnapshotId());
return createFileStatus(fsd, HdfsFileStatus.EMPTY_NAME, nodeAttrs,
- policyId, iip.getPathSnapshotId(), isRawPath, iip);
+ policyId, iip);
} finally {
fsd.readUnlock();
}
}
- static HdfsFileStatus getFileInfo(
- FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath)
+ static HdfsFileStatus getFileInfo(FSDirectory fsd, INodesInPath iip)
throws IOException {
- fsd.readLock();
- try {
- HdfsFileStatus status = null;
- final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
- if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
- status = FSDirectory.DOT_RESERVED_STATUS;
- } else if (iip.isDotSnapshotDir()) {
- if (fsd.getINode4DotSnapshot(iip) != null) {
- status = FSDirectory.DOT_SNAPSHOT_DIR_STATUS;
- }
- } else {
- status = getFileInfo(fsd, src, iip, isRawPath, true);
+ if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
+ return FSDirectory.DOT_RESERVED_STATUS;
+ }
+
+ if (iip.isDotSnapshotDir()) {
+ if (fsd.getINode4DotSnapshot(iip) != null) {
+ return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
+ HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
+ HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
}
- return status;
- } finally {
- fsd.readUnlock();
+ return null;
}
+
+ return getFileInfo(fsd, iip, true);
}
/**
@@ -423,15 +415,12 @@ class FSDirStatAndListingOp {
*/
private static HdfsFileStatus createFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath,
- INodesInPath iip)
+ boolean needLocation, byte storagePolicy, INodesInPath iip)
throws IOException {
if (needLocation) {
- return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy,
- snapshot, isRawPath, iip);
+ return createLocatedFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
} else {
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
- snapshot, isRawPath, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
}
}
@@ -445,8 +434,7 @@ class FSDirStatAndListingOp {
INodesInPath iip) throws IOException {
INodeAttributes nodeAttrs = getINodeAttributes(
fsd, fullPath, path, iip.getLastINode(), snapshot);
- return createFileStatus(fsd, path, nodeAttrs, storagePolicy,
- snapshot, isRawPath, iip);
+ return createFileStatus(fsd, path, nodeAttrs, storagePolicy, iip);
}
/**
@@ -454,14 +442,15 @@ class FSDirStatAndListingOp {
* @param iip the INodesInPath containing the target INode and its ancestors
*/
static HdfsFileStatus createFileStatus(
- FSDirectory fsd, byte[] path,
- INodeAttributes nodeAttrs, byte storagePolicy, int snapshot,
- boolean isRawPath, INodesInPath iip) throws IOException {
+ FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
+ byte storagePolicy, INodesInPath iip) throws IOException {
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
final boolean isEncrypted;
final INode node = iip.getLastINode();
+ final int snapshot = iip.getPathSnapshotId();
+ final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
@@ -511,10 +500,9 @@ class FSDirStatAndListingOp {
* Create FileStatus with location info by file INode
* @param iip the INodesInPath containing the target INode and its ancestors
*/
- private static HdfsLocatedFileStatus createLocatedFileStatus(
+ private static HdfsFileStatus createLocatedFileStatus(
FSDirectory fsd, byte[] path, INodeAttributes nodeAttrs,
- byte storagePolicy, int snapshot,
- boolean isRawPath, INodesInPath iip) throws IOException {
+ byte storagePolicy, INodesInPath iip) throws IOException {
assert fsd.hasReadLock();
long size = 0; // length is zero for directories
short replication = 0;
@@ -522,6 +510,8 @@ class FSDirStatAndListingOp {
LocatedBlocks loc = null;
final boolean isEncrypted;
final INode node = iip.getLastINode();
+ final int snapshot = iip.getPathSnapshotId();
+ final boolean isRawPath = iip.isRaw();
final FileEncryptionInfo feInfo = isRawPath ? null : FSDirEncryptionZoneOp
.getFileEncryptionInfo(fsd, node, snapshot, iip);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 8429024..030d8cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -340,7 +340,6 @@ class FSDirWriteFileOp {
version = ezInfo.protocolVersion;
}
- boolean isRawPath = FSDirectory.isReservedRawName(src);
FSDirectory fsd = fsn.getFSDirectory();
INodesInPath iip = fsd.resolvePathForWrite(pc, src);
src = iip.getPath();
@@ -444,7 +443,7 @@ class FSDirWriteFileOp {
NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
src + " inode " + newNode.getId() + " " + holder);
}
- return FSDirStatAndListingOp.getFileInfo(fsd, src, false, isRawPath);
+ return FSDirStatAndListingOp.getFileInfo(fsd, iip);
}
static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index b6652e4..5dc5a03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -531,21 +531,24 @@ public class FSDirectory implements Closeable {
* @throws FileNotFoundException
* @throws AccessControlException
*/
- INodesInPath resolvePath(FSPermissionChecker pc, String src)
+ @VisibleForTesting
+ public INodesInPath resolvePath(FSPermissionChecker pc, String src)
throws UnresolvedLinkException, FileNotFoundException,
AccessControlException {
return resolvePath(pc, src, true);
}
- INodesInPath resolvePath(FSPermissionChecker pc, String src,
+ @VisibleForTesting
+ public INodesInPath resolvePath(FSPermissionChecker pc, String src,
boolean resolveLink) throws UnresolvedLinkException,
FileNotFoundException, AccessControlException {
byte[][] components = INode.getPathComponents(src);
- if (isPermissionEnabled && pc != null && isReservedRawName(components)) {
+ boolean isRaw = isReservedRawName(components);
+ if (isPermissionEnabled && pc != null && isRaw) {
pc.checkSuperuserPrivilege();
}
components = resolveComponents(components, this);
- return INodesInPath.resolve(rootDir, components, resolveLink);
+ return INodesInPath.resolve(rootDir, components, isRaw, resolveLink);
}
INodesInPath resolvePathForWrite(FSPermissionChecker pc, String src)
@@ -1662,8 +1665,7 @@ public class FSDirectory implements Closeable {
HdfsFileStatus getAuditFileInfo(INodesInPath iip)
throws IOException {
return (namesystem.isAuditEnabled() && namesystem.isExternalInvocation())
- ? FSDirStatAndListingOp.getFileInfo(this, iip.getPath(), iip, false,
- false) : null;
+ ? FSDirStatAndListingOp.getFileInfo(this, iip, false) : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index 86cab28..af8998f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -126,6 +126,12 @@ public class INodesInPath {
static INodesInPath resolve(final INodeDirectory startingDir,
final byte[][] components, final boolean resolveLink)
throws UnresolvedLinkException {
+ return resolve(startingDir, components, false, resolveLink);
+ }
+
+ static INodesInPath resolve(final INodeDirectory startingDir,
+ final byte[][] components, final boolean isRaw,
+ final boolean resolveLink) throws UnresolvedLinkException {
Preconditions.checkArgument(startingDir.compareTo(components[0]) == 0);
INode curNode = startingDir;
@@ -225,7 +231,7 @@ public class INodesInPath {
System.arraycopy(inodes, 0, newNodes, 0, newNodes.length);
inodes = newNodes;
}
- return new INodesInPath(inodes, components, isSnapshot, snapshotId);
+ return new INodesInPath(inodes, components, isRaw, isSnapshot, snapshotId);
}
private static boolean shouldUpdateLatestId(int sid, int snapshotId) {
@@ -249,7 +255,8 @@ public class INodesInPath {
INode[] inodes = new INode[iip.inodes.length];
System.arraycopy(iip.inodes, 0, inodes, 0, inodes.length);
inodes[pos] = inode;
- return new INodesInPath(inodes, iip.path, iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, iip.path, iip.isRaw,
+ iip.isSnapshot, iip.snapshotId);
}
/**
@@ -267,7 +274,8 @@ public class INodesInPath {
byte[][] path = new byte[iip.path.length + 1][];
System.arraycopy(iip.path, 0, path, 0, path.length - 1);
path[path.length - 1] = childName;
- return new INodesInPath(inodes, path, iip.isSnapshot, iip.snapshotId);
+ return new INodesInPath(inodes, path, iip.isRaw,
+ iip.isSnapshot, iip.snapshotId);
}
private final byte[][] path;
@@ -279,6 +287,13 @@ public class INodesInPath {
* true if this path corresponds to a snapshot
*/
private final boolean isSnapshot;
+
+ /**
+ * true if this is a /.reserved/raw path. path component resolution strips
+ * it from the path so need to track it separately.
+ */
+ private final boolean isRaw;
+
/**
* For snapshot paths, it is the id of the snapshot; or
* {@link Snapshot#CURRENT_STATE_ID} if the snapshot does not exist. For
@@ -287,17 +302,18 @@ public class INodesInPath {
*/
private final int snapshotId;
- private INodesInPath(INode[] inodes, byte[][] path, boolean isSnapshot,
- int snapshotId) {
+ private INodesInPath(INode[] inodes, byte[][] path, boolean isRaw,
+ boolean isSnapshot,int snapshotId) {
Preconditions.checkArgument(inodes != null && path != null);
this.inodes = inodes;
this.path = path;
+ this.isRaw = isRaw;
this.isSnapshot = isSnapshot;
this.snapshotId = snapshotId;
}
private INodesInPath(INode[] inodes, byte[][] path) {
- this(inodes, path, false, CURRENT_STATE_ID);
+ this(inodes, path, false, false, CURRENT_STATE_ID);
}
/**
@@ -400,7 +416,7 @@ public class INodesInPath {
final byte[][] apath = new byte[length][];
System.arraycopy(this.inodes, 0, anodes, 0, length);
System.arraycopy(this.path, 0, apath, 0, length);
- return new INodesInPath(anodes, apath, false, snapshotId);
+ return new INodesInPath(anodes, apath, isRaw, false, snapshotId);
}
/**
@@ -428,7 +444,7 @@ public class INodesInPath {
byte[][] existingPath = new byte[i][];
System.arraycopy(inodes, 0, existing, 0, i);
System.arraycopy(path, 0, existingPath, 0, i);
- return new INodesInPath(existing, existingPath, false, snapshotId);
+ return new INodesInPath(existing, existingPath, isRaw, false, snapshotId);
}
/**
@@ -438,10 +454,20 @@ public class INodesInPath {
return this.isSnapshot;
}
+ /**
+ * @return if .snapshot is the last path component.
+ */
boolean isDotSnapshotDir() {
return isDotSnapshotDir(getLastLocalName());
}
+ /**
+ * @return if this is a /.reserved/raw path.
+ */
+ public boolean isRaw() {
+ return isRaw;
+ }
+
private static String toString(INode inode) {
return inode == null? null: inode.getLocalName();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22fc46d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
index 449f715..c09d346 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
+import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
@@ -49,6 +50,8 @@ import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesNotEqual;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.apache.hadoop.test.GenericTestUtils.assertMatches;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestReservedRawPaths {
@@ -100,6 +103,24 @@ public class TestReservedRawPaths {
}
/**
+ * Verify resolving path will return an iip that tracks if the original
+ * path was a raw path.
+ */
+ @Test(timeout = 120000)
+ public void testINodesInPath() throws IOException {
+ FSDirectory fsd = cluster.getNamesystem().getFSDirectory();
+ final String path = "/path";
+
+ INodesInPath iip = fsd.resolvePath(null, path);
+ assertFalse(iip.isRaw());
+ assertEquals(path, iip.getPath());
+
+ iip = fsd.resolvePath(null, "/.reserved/raw" + path);
+ assertTrue(iip.isRaw());
+ assertEquals(path, iip.getPath());
+ }
+
+ /**
* Basic read/write tests of raw files.
* Create a non-encrypted file
* Create an encryption zone
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/52] [abbrv] hadoop git commit: HDFS-10692. Update JDiff report's
base version for HDFS from 2.6.0 to 2.7.2. Contributed by Wangda Tan.
Posted by aw...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc7a1c54/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.7.2.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.7.2.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.7.2.xml
new file mode 100644
index 0000000..028ba2d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.7.2.xml
@@ -0,0 +1,21505 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Thu Aug 18 16:02:32 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="Apache Hadoop HDFS 2.7.2"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/classes:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.7.2.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/lib/tools.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.7.2.jar:/Users/wtan/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar:/Users/wtan/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/wtan/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/http
core-4.2.5.jar:/Users/wtan/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/wtan/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/wtan/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/wtan/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/wtan/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/wtan/.m2/repository/jline/jline/0.9.94/jline-0.9.94.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/hadoop-common-2.7.2.jar:/Users/wtan/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/wtan/.m2/repository/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/
wtan/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/wtan/.m2/repository/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/Users/wtan/.m2/repository/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/wtan/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/wtan/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/wtan/.m2/repository/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/Users/wtan/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/wtan/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/wtan/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/wtan/.m2/repos
itory/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/Users/wtan/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/wtan/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/wtan/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/wtan/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/wtan/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/wtan/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/wtan/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar:/Users/wtan/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/wtan/.m2/repository/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar:/Users/wtan/.m2/repository/org/apache/curator/curat
or-recipes/2.7.1/curator-recipes-2.7.1.jar:/Users/wtan/.m2/repository/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar:/Users/wtan/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/wtan/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar:/Users/wtan/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/wtan/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/wtan/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/wtan/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/wtan/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/wtan/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/wtan/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/wtan/.m2/repository/commons-lang/common
s-lang/2.6/commons-lang-2.6.jar:/Users/wtan/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/wtan/.m2/repository/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/Users/wtan/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/wtan/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/wtan/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/wtan/.m2/repository/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/wtan/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/wtan/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/wtan/.m2/repository/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar:/Users/wtan/.m2/repository/xerces/xercesIm
pl/2.9.1/xercesImpl-2.9.1.jar:/Users/wtan/.m2/repository/xml-apis/xml-apis/1.3.04/xml-apis-1.3.04.jar:/Users/wtan/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar:/Users/wtan/.m2/repository/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar -sourcepath /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -apidir /Users/wtan/project/github/hadoop-common-trunk/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 2.7.2 -->
+<package name="org.apache.hadoop.fs">
+ <!-- start class org.apache.hadoop.fs.BlockStorageLocation -->
+ <class name="BlockStorageLocation" extends="org.apache.hadoop.fs.BlockLocation"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="BlockStorageLocation" type="org.apache.hadoop.fs.BlockLocation, org.apache.hadoop.fs.VolumeId[]"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getVolumeIds" return="org.apache.hadoop.fs.VolumeId[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Gets the list of {@link VolumeId} corresponding to the block's replicas.
+
+ @return volumeIds list of VolumeId for the block's replicas]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Wrapper for {@link BlockLocation} that also adds {@link VolumeId} volume
+ location information for each replica.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.fs.BlockStorageLocation -->
+ <!-- start class org.apache.hadoop.fs.CacheFlag -->
+ <class name="CacheFlag" extends="java.lang.Enum"
+ abstract="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.fs.CacheFlag[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.fs.CacheFlag"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ <doc>
+ <![CDATA[Specifies semantics for CacheDirective operations. Multiple flags can
+ be combined in an EnumSet.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.fs.CacheFlag -->
+ <!-- start class org.apache.hadoop.fs.HdfsVolumeId -->
+ <class name="HdfsVolumeId" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.fs.VolumeId"/>
+ <constructor name="HdfsVolumeId" type="byte[]"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="compareTo" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.fs.VolumeId"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="obj" type="java.lang.Object"/>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[HDFS-specific volume identifier which implements {@link VolumeId}. Can be
+ used to differentiate between the data directories on a single datanode. This
+ identifier is only unique on a per-datanode basis.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.fs.HdfsVolumeId -->
+ <!-- start interface org.apache.hadoop.fs.VolumeId -->
+ <interface name="VolumeId" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="java.lang.Comparable"/>
+ <method name="compareTo" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="arg0" type="org.apache.hadoop.fs.VolumeId"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="obj" type="java.lang.Object"/>
+ </method>
+ <doc>
+ <![CDATA[Opaque interface that identifies a disk location. Subclasses
+ should implement {@link Comparable} and override both equals and hashCode.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.fs.VolumeId -->
+ <!-- start class org.apache.hadoop.fs.XAttr.Builder -->
+ <class name="XAttr.Builder" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Builder"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setNameSpace" return="org.apache.hadoop.fs.XAttr.Builder"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ns" type="org.apache.hadoop.fs.XAttr.NameSpace"/>
+ </method>
+ <method name="setName" return="org.apache.hadoop.fs.XAttr.Builder"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ <method name="setValue" return="org.apache.hadoop.fs.XAttr.Builder"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="value" type="byte[]"/>
+ </method>
+ <method name="build" return="org.apache.hadoop.fs.XAttr"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.fs.XAttr.Builder -->
+ <!-- start class org.apache.hadoop.fs.XAttr.NameSpace -->
+ <class name="XAttr.NameSpace" extends="java.lang.Enum"
+ abstract="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="values" return="org.apache.hadoop.fs.XAttr.NameSpace[]"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="valueOf" return="org.apache.hadoop.fs.XAttr.NameSpace"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.fs.XAttr.NameSpace -->
+</package>
+<package name="org.apache.hadoop.hdfs">
+ <!-- start interface org.apache.hadoop.hdfs.BlockReader -->
+ <interface name="BlockReader" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.fs.ByteBufferReadable"/>
+ <method name="read" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="buf" type="byte[]"/>
+ <param name="off" type="int"/>
+ <param name="len" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="skip" return="long"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="n" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Skip the given number of bytes]]>
+ </doc>
+ </method>
+ <method name="available" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns an estimate of the number of bytes that can be read
+ (or skipped over) from this input stream without performing
+ network I/O.
+ This may return more than what is actually present in the block.]]>
+ </doc>
+ </method>
+ <method name="close"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Close the block reader.
+
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="readFully"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="buf" type="byte[]"/>
+ <param name="readOffset" type="int"/>
+ <param name="amtToRead" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Read exactly the given amount of data, throwing an exception
+ if EOF is reached before that amount]]>
+ </doc>
+ </method>
+ <method name="readAll" return="int"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="buf" type="byte[]"/>
+ <param name="offset" type="int"/>
+ <param name="len" type="int"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Similar to {@link #readFully(byte[], int, int)} except that it will
+ not throw an exception on EOF. However, it differs from the simple
+ {@link #read(byte[], int, int)} call in that it is guaranteed to
+ read the data if it is available. In other words, if this call
+ does not throw an exception, then either the buffer has been
+ filled or the next call will return EOF.]]>
+ </doc>
+ </method>
+ <method name="isLocal" return="boolean"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return true only if this is a local read.]]>
+ </doc>
+ </method>
+ <method name="isShortCircuit" return="boolean"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return true only if this is a short-circuit read.
+ All short-circuit reads are also local.]]>
+ </doc>
+ </method>
+ <method name="getClientMmap" return="org.apache.hadoop.hdfs.shortcircuit.ClientMmap"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="opts" type="java.util.EnumSet"/>
+ <doc>
+ <![CDATA[Get a ClientMmap object for this BlockReader.
+
+ @param opts The read options to use.
+ @return The ClientMmap object, or null if mmap is not
+ supported.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[A BlockReader is responsible for reading a single block
+ from a single datanode.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.hdfs.BlockReader -->
+ <!-- start class org.apache.hadoop.hdfs.BlockReaderFactory.BlockReaderPeer -->
+ <class name="BlockReaderFactory.BlockReaderPeer" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.BlockReaderFactory.BlockReaderPeer -->
+ <!-- start class org.apache.hadoop.hdfs.BlockReaderFactory.FailureInjector -->
+ <class name="BlockReaderFactory.FailureInjector" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="FailureInjector"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="injectRequestFileDescriptorsFailure"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getSupportsReceiptVerification" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.BlockReaderFactory.FailureInjector -->
+ <!-- start class org.apache.hadoop.hdfs.CorruptFileBlockIterator -->
+ <class name="CorruptFileBlockIterator" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.fs.RemoteIterator"/>
+ <constructor name="CorruptFileBlockIterator" type="org.apache.hadoop.hdfs.DFSClient, org.apache.hadoop.fs.Path"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getCallsMade" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return the number of calls made to the DFSClient.
+ This is for debugging and testing purposes.]]>
+ </doc>
+ </method>
+ <method name="hasNext" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="next" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <doc>
+ <![CDATA[Provides an iterator interface for listCorruptFileBlocks.
+ This class is used by DistributedFileSystem and Hdfs.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.CorruptFileBlockIterator -->
+ <!-- start class org.apache.hadoop.hdfs.DFSClient.Conf -->
+ <class name="DFSClient.Conf" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Conf" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="isUseLegacyBlockReaderLocal" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDomainSocketPath" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="isShortCircuitLocalReads" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="isDomainSocketDataTraffic" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <field name="brfFailureInjector" type="org.apache.hadoop.hdfs.BlockReaderFactory.FailureInjector"
+ transient="false" volatile="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[DFSClient configuration]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSClient.Conf -->
+ <!-- start class org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream -->
+ <class name="DFSClient.DFSDataInputStream" extends="org.apache.hadoop.hdfs.client.HdfsDataInputStream"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="use {@link HdfsDataInputStream} instead.">
+ <constructor name="DFSDataInputStream" type="org.apache.hadoop.hdfs.DFSInputStream"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <doc>
+ <![CDATA[@deprecated use {@link HdfsDataInputStream} instead.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream -->
+ <!-- start class org.apache.hadoop.hdfs.DFSHedgedReadMetrics -->
+ <class name="DFSHedgedReadMetrics" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="DFSHedgedReadMetrics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="incHedgedReadOps"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="incHedgedReadOpsInCurThread"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="incHedgedReadWins"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getHedgedReadOps" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getHedgedReadOpsInCurThread" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getHedgedReadWins" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <field name="hedgedReadOps" type="java.util.concurrent.atomic.AtomicLong"
+ transient="false" volatile="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="hedgedReadOpsWin" type="java.util.concurrent.atomic.AtomicLong"
+ transient="false" volatile="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <field name="hedgedReadOpsInCurThread" type="java.util.concurrent.atomic.AtomicLong"
+ transient="false" volatile="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[The client-side metrics for hedged read feature.
+ This class has a number of metrics variables that are publicly accessible,
+ we can grab them from client side, like HBase.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSHedgedReadMetrics -->
+ <!-- start class org.apache.hadoop.hdfs.DFSInotifyEventInputStream -->
+ <class name="DFSInotifyEventInputStream" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="poll" return="org.apache.hadoop.hdfs.inotify.EventBatch"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="MissingEventsException" type="org.apache.hadoop.hdfs.inotify.MissingEventsException"/>
+ <doc>
+ <![CDATA[Returns the next batch of events in the stream or null if no new
+ batches are currently available.
+
+ @throws IOException because of network error or edit log
+ corruption. Also possible if JournalNodes are unresponsive in the
+ QJM setting (even one unresponsive JournalNode is enough in rare cases),
+ so catching this exception and retrying at least a few times is
+ recommended.
+ @throws MissingEventsException if we cannot return the next batch in the
+ stream because the data for the events (and possibly some subsequent
+ events) has been deleted (generally because this stream is a very large
+ number of transactions behind the current state of the NameNode). It is
+ safe to continue reading from the stream after this exception is thrown
+ The next available batch of events will be returned.]]>
+ </doc>
+ </method>
+ <method name="getTxidsBehindEstimate" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return a estimate of how many transaction IDs behind the NameNode's
+ current state this stream is. Clients should periodically call this method
+ and check if its result is steadily increasing, which indicates that they
+ are falling behind (i.e. transaction are being generated faster than the
+ client is reading them). If a client falls too far behind events may be
+ deleted before the client can read them.
+ <p/>
+ A return value of -1 indicates that an estimate could not be produced, and
+ should be ignored. The value returned by this method is really only useful
+ when compared to previous or subsequent returned values.]]>
+ </doc>
+ </method>
+ <method name="poll" return="org.apache.hadoop.hdfs.inotify.EventBatch"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="time" type="long"/>
+ <param name="tu" type="java.util.concurrent.TimeUnit"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ <exception name="MissingEventsException" type="org.apache.hadoop.hdfs.inotify.MissingEventsException"/>
+ <doc>
+ <![CDATA[Returns the next event batch in the stream, waiting up to the specified
+ amount of time for a new batch. Returns null if one is not available at the
+ end of the specified amount of time. The time before the method returns may
+ exceed the specified amount of time by up to the time required for an RPC
+ to the NameNode.
+
+ @param time number of units of the given TimeUnit to wait
+ @param tu the desired TimeUnit
+ @throws IOException see {@link DFSInotifyEventInputStream#poll()}
+ @throws MissingEventsException
+ see {@link DFSInotifyEventInputStream#poll()}
+ @throws InterruptedException if the calling thread is interrupted]]>
+ </doc>
+ </method>
+ <method name="take" return="org.apache.hadoop.hdfs.inotify.EventBatch"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="InterruptedException" type="java.lang.InterruptedException"/>
+ <exception name="MissingEventsException" type="org.apache.hadoop.hdfs.inotify.MissingEventsException"/>
+ <doc>
+ <![CDATA[Returns the next batch of events in the stream, waiting indefinitely if
+ a new batch is not immediately available.
+
+ @throws IOException see {@link DFSInotifyEventInputStream#poll()}
+ @throws MissingEventsException see
+ {@link DFSInotifyEventInputStream#poll()}
+ @throws InterruptedException if the calling thread is interrupted]]>
+ </doc>
+ </method>
+ <field name="LOG" type="org.slf4j.Logger"
+ transient="false" volatile="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ <doc>
+ <![CDATA[Stream for reading inotify events. DFSInotifyEventInputStreams should not
+ be shared among multiple threads.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSInotifyEventInputStream -->
+ <!-- start class org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics -->
+ <class name="DFSInputStream.ReadStatistics" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ReadStatistics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="ReadStatistics" type="org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getTotalBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total bytes read. This will always be at least as
+ high as the other numbers, since it includes all of them.]]>
+ </doc>
+ </method>
+ <method name="getTotalLocalBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total local bytes read. This will always be at least
+ as high as totalShortCircuitBytesRead, since all short-circuit
+ reads are also local.]]>
+ </doc>
+ </method>
+ <method name="getTotalShortCircuitBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total short-circuit local bytes read.]]>
+ </doc>
+ </method>
+ <method name="getTotalZeroCopyBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total number of zero-copy bytes read.]]>
+ </doc>
+ </method>
+ <method name="getRemoteBytesRead" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return The total number of bytes read which were not local.]]>
+ </doc>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSInputStream.ReadStatistics -->
+ <!-- start class org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress -->
+ <class name="DFSUtil.ConfiguredNNAddress" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="getNameserviceId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getNamenodeId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getAddress" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Represent one of the NameNodes configured in the cluster.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress -->
+ <!-- start class org.apache.hadoop.hdfs.ExtendedBlockId -->
+ <class name="ExtendedBlockId" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ExtendedBlockId" type="long, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="fromExtendedBlock" return="org.apache.hadoop.hdfs.ExtendedBlockId"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="block" type="org.apache.hadoop.hdfs.protocol.ExtendedBlock"/>
+ </method>
+ <method name="getBlockId" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getBlockPoolId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="equals" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="o" type="java.lang.Object"/>
+ </method>
+ <method name="hashCode" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[An immutable key which identifies a block.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.ExtendedBlockId -->
+ <!-- start class org.apache.hadoop.hdfs.HAUtil -->
+ <class name="HAUtil" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="isHAEnabled" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Returns true if HA for namenode is configured for the given nameservice
+
+ @param conf Configuration
+ @param nsId nameservice, or null if no federated NS is configured
+ @return true if HA is configured in the configuration; else false.]]>
+ </doc>
+ </method>
+ <method name="usesSharedEditsDir" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Returns true if HA is using a shared edits directory.
+
+ @param conf Configuration
+ @return true if HA config is using a shared edits dir, false otherwise.]]>
+ </doc>
+ </method>
+ <method name="getNameNodeId" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the namenode Id by matching the {@code addressKey}
+ with the the address of the local node.
+
+ If {@link DFSConfigKeys#DFS_HA_NAMENODE_ID_KEY} is not specifically
+ configured, this method determines the namenode Id by matching the local
+ node's address with the configured addresses. When a match is found, it
+ returns the namenode Id from the corresponding configuration key.
+
+ @param conf Configuration
+ @return namenode Id on success, null on failure.
+ @throws HadoopIllegalArgumentException on error]]>
+ </doc>
+ </method>
+ <method name="getNameNodeIdFromAddress" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="address" type="java.net.InetSocketAddress"/>
+ <param name="keys" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Similar to
+ {@link DFSUtil#getNameServiceIdFromAddress(Configuration,
+ InetSocketAddress, String...)}]]>
+ </doc>
+ </method>
+ <method name="getNameNodeIdOfOtherNode" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the NN ID of the other node in an HA setup.
+
+ @param conf the configuration of this node
+ @return the NN ID of the other node in this nameservice]]>
+ </doc>
+ </method>
+ <method name="getConfForOtherNode" return="org.apache.hadoop.conf.Configuration"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="myConf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Given the configuration for this node, return a Configuration object for
+ the other node in an HA setup.
+
+ @param myConf the configuration of this node
+ @return the configuration of the other node in an HA setup]]>
+ </doc>
+ </method>
+ <method name="shouldAllowStandbyReads" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[This is used only by tests at the moment.
+ @return true if the NN should allow read operations while in standby mode.]]>
+ </doc>
+ </method>
+ <method name="setAllowStandbyReads"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="val" type="boolean"/>
+ </method>
+ <method name="isLogicalUri" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <doc>
+ <![CDATA[@return true if the given nameNodeUri appears to be a logical URI.]]>
+ </doc>
+ </method>
+ <method name="isClientFailoverConfigured" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <doc>
+ <![CDATA[Check whether the client has a failover proxy provider configured
+ for the namenode/nameservice.
+
+ @param conf Configuration
+ @param nameNodeUri The URI of namenode
+ @return true if failover is configured.]]>
+ </doc>
+ </method>
+ <method name="useLogicalUri" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Check whether logical URI is needed for the namenode and
+ the corresponding failover proxy provider in the config.
+
+ @param conf Configuration
+ @param nameNodeUri The URI of namenode
+ @return true if logical URI is needed. false, if not needed.
+ @throws IOException most likely due to misconfiguration.]]>
+ </doc>
+ </method>
+ <method name="getServiceUriFromToken" return="java.net.URI"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="scheme" type="java.lang.String"/>
+ <param name="token" type="org.apache.hadoop.security.token.Token"/>
+ <doc>
+ <![CDATA[Parse the file system URI out of the provided token.]]>
+ </doc>
+ </method>
+ <method name="buildTokenServiceForLogicalUri" return="org.apache.hadoop.io.Text"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="uri" type="java.net.URI"/>
+ <param name="scheme" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the service name used in the delegation token for the given logical
+ HA service.
+ @param uri the logical URI of the cluster
+ @param scheme the scheme of the corresponding FileSystem
+ @return the service name]]>
+ </doc>
+ </method>
+ <method name="isTokenForLogicalUri" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="token" type="org.apache.hadoop.security.token.Token"/>
+ <doc>
+ <![CDATA[@return true if this token corresponds to a logical nameservice
+ rather than a specific namenode.]]>
+ </doc>
+ </method>
+ <method name="buildTokenServicePrefixForLogicalUri" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="scheme" type="java.lang.String"/>
+ </method>
+ <method name="cloneDelegationTokenForLogicalUri"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="haUri" type="java.net.URI"/>
+ <param name="nnAddrs" type="java.util.Collection"/>
+ <doc>
+ <![CDATA[Locate a delegation token associated with the given HA cluster URI, and if
+ one is found, clone it to also represent the underlying namenode address.
+ @param ugi the UGI to modify
+ @param haUri the logical URI for the cluster
+ @param nnAddrs collection of NNs in the cluster to which the token
+ applies]]>
+ </doc>
+ </method>
+ <method name="getAddressOfActive" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="fs" type="org.apache.hadoop.fs.FileSystem"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the internet address of the currently-active NN. This should rarely be
+ used, since callers of this method who connect directly to the NN using the
+ resulting InetSocketAddress will not be able to connect to the active NN if
+ a failover were to occur after this method has been called.
+
+ @param fs the file system to get the active address of.
+ @return the internet address of the currently-active NN.
+ @throws IOException if an error occurs while resolving the active NN.]]>
+ </doc>
+ </method>
+ <method name="getProxiesForAllNameNodesInNameservice" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
+ call should be made on every NN in an HA nameservice, not just the active.
+
+ @param conf configuration
+ @param nsId the nameservice to get all of the proxies for.
+ @return a list of RPC proxies for each NN in the nameservice.
+ @throws IOException in the event of error.]]>
+ </doc>
+ </method>
+ <method name="getProxiesForAllNameNodesInNameservice" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nsId" type="java.lang.String"/>
+ <param name="xface" type="java.lang.Class"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
+ call should be made on every NN in an HA nameservice, not just the active.
+
+ @param conf configuration
+ @param nsId the nameservice to get all of the proxies for.
+ @param xface the protocol class.
+ @return a list of RPC proxies for each NN in the nameservice.
+ @throws IOException in the event of error.]]>
+ </doc>
+ </method>
+ <method name="isAtLeastOneActive" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="namenodes" type="java.util.List"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Used to ensure that at least one of the given HA NNs is currently in the
+ active state..
+
+ @param namenodes list of RPC proxies for each NN to check.
+ @return true if at least one NN is active, false if all are in the standby state.
+ @throws IOException in the event of error.]]>
+ </doc>
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.HAUtil -->
+ <!-- start class org.apache.hadoop.hdfs.KeyProviderCache -->
+ <class name="KeyProviderCache" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="KeyProviderCache" type="long"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="get" return="org.apache.hadoop.crypto.key.KeyProvider"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="setKeyProvider"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="keyProvider" type="org.apache.hadoop.crypto.key.KeyProvider"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <field name="LOG" type="org.apache.commons.logging.Log"
+ transient="false" volatile="false"
+ static="true" final="true" visibility="public"
+ deprecated="not deprecated">
+ </field>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.KeyProviderCache -->
+ <!-- start class org.apache.hadoop.hdfs.NameNodeProxies -->
+ <class name="NameNodeProxies" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="NameNodeProxies"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="createProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates the namenode proxy with the passed protocol. This will handle
+ creation of either HA- or non-HA-enabled proxy objects, depending upon
+ if the provided URI is a configured logical URI.
+
+ @param conf the configuration containing the required IPC
+ properties, client failover configurations, etc.
+ @param nameNodeUri the URI pointing either to a specific NameNode
+ or to a logical nameservice.
+ @param xface the IPC interface which should be created
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException if there is an error creating the proxy]]>
+ </doc>
+ </method>
+ <method name="createProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates the namenode proxy with the passed protocol. This will handle
+ creation of either HA- or non-HA-enabled proxy objects, depending upon
+ if the provided URI is a configured logical URI.
+
+ @param conf the configuration containing the required IPC
+ properties, client failover configurations, etc.
+ @param nameNodeUri the URI pointing either to a specific NameNode
+ or to a logical nameservice.
+ @param xface the IPC interface which should be created
+ @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ a secure client falls back to simple auth
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException if there is an error creating the proxy]]>
+ </doc>
+ </method>
+ <method name="createProxyWithLossyRetryHandler" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="numResponseToDrop" type="int"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Generate a dummy namenode proxy instance that utilizes our hacked
+ {@link LossyRetryInvocationHandler}. Proxy instance generated using this
+ method will proactively drop RPC responses. Currently this method only
+ support HA setup. null will be returned if the given configuration is not
+ for HA.
+
+ @param config the configuration containing the required IPC
+ properties, client failover configurations, etc.
+ @param nameNodeUri the URI pointing either to a specific NameNode
+ or to a logical nameservice.
+ @param xface the IPC interface which should be created
+ @param numResponseToDrop The number of responses to drop for each RPC call
+ @param fallbackToSimpleAuth set to true or false during calls to indicate if
+ a secure client falls back to simple auth
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to. Will return null of the
+ given configuration does not support HA.
+ @throws IOException if there is an error creating the proxy]]>
+ </doc>
+ </method>
+ <method name="createNonHAProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nnAddr" type="java.net.InetSocketAddress"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="withRetries" type="boolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates an explicitly non-HA-enabled proxy object. Most of the time you
+ don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
+
+ @param conf the configuration object
+ @param nnAddr address of the remote NN to connect to
+ @param xface the IPC interface which should be created
+ @param ugi the user who is making the calls on the proxy object
+ @param withRetries certain interfaces have a non-standard retry policy
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="createNonHAProxy" return="org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nnAddr" type="java.net.InetSocketAddress"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+ <param name="withRetries" type="boolean"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates an explicitly non-HA-enabled proxy object. Most of the time you
+ don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
+
+ @param conf the configuration object
+ @param nnAddr address of the remote NN to connect to
+ @param xface the IPC interface which should be created
+ @param ugi the user who is making the calls on the proxy object
+ @param withRetries certain interfaces have a non-standard retry policy
+ @param fallbackToSimpleAuth - set to true or false during this method to
+ indicate if a secure client falls back to simple auth
+ @return an object containing both the proxy and the associated
+ delegation token service it corresponds to
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getFailoverProxyProviderClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Gets the configured Failover proxy provider's class]]>
+ </doc>
+ </method>
+ <method name="createFailoverProxyProvider" return="org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="nameNodeUri" type="java.net.URI"/>
+ <param name="xface" type="java.lang.Class"/>
+ <param name="checkPort" type="boolean"/>
+ <param name="fallbackToSimpleAuth" type="java.util.concurrent.atomic.AtomicBoolean"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Creates the Failover proxy provider instance]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Create proxy objects to communicate with a remote NN. All remote access to an
+ NN should be funneled through this class. Most of the time you'll want to use
+ {@link NameNodeProxies#createProxy(Configuration, URI, Class)}, which will
+ create either an HA- or non-HA-enabled client proxy as appropriate.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.NameNodeProxies -->
+ <!-- start class org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo -->
+ <class name="NameNodeProxies.ProxyAndInfo" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ProxyAndInfo" type="PROXYTYPE, org.apache.hadoop.io.Text, java.net.InetSocketAddress"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getProxy" return="PROXYTYPE"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getDelegationTokenService" return="org.apache.hadoop.io.Text"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getAddress" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Wrapper for a client proxy as well as its associated service ID.
+ This is simply used as a tuple-like return type for
+ {@link NameNodeProxies#createProxy} and
+ {@link NameNodeProxies#createNonHAProxy}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo -->
+ <!-- start interface org.apache.hadoop.hdfs.RemotePeerFactory -->
+ <interface name="RemotePeerFactory" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="newConnectedPeer" return="org.apache.hadoop.hdfs.net.Peer"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <param name="blockToken" type="org.apache.hadoop.security.token.Token"/>
+ <param name="datanodeId" type="org.apache.hadoop.hdfs.protocol.DatanodeID"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[@param addr The address to connect to.
+ @param blockToken Token used during optional SASL negotiation
+ @param datanodeId ID of destination DataNode
+ @return A new Peer connected to the address.
+
+ @throws IOException If there was an error connecting or creating
+ the remote socket, encrypted stream, etc.]]>
+ </doc>
+ </method>
+ </interface>
+ <!-- end interface org.apache.hadoop.hdfs.RemotePeerFactory -->
+ <!-- start class org.apache.hadoop.hdfs.UnknownCipherSuiteException -->
+ <class name="UnknownCipherSuiteException" extends="java.io.IOException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="UnknownCipherSuiteException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <doc>
+ <![CDATA[Thrown when an unknown cipher suite is encountered.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.UnknownCipherSuiteException -->
+ <!-- start class org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException -->
+ <class name="UnknownCryptoProtocolVersionException" extends="java.io.IOException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="UnknownCryptoProtocolVersionException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="UnknownCryptoProtocolVersionException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException -->
+ <doc>
+ <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}. This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files
+have strictly one writer at any one time. Bytes are always appended
+to the end of the writer's stream. There is no notion of "record appends"
+or "mutations" that are then checked or reordered. Writers simply emit
+a byte stream. That byte stream is guaranteed to be stored in the
+order written.</p>]]>
+ </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.client">
+ <!-- start class org.apache.hadoop.hdfs.client.BlockReportOptions -->
+ <class name="BlockReportOptions" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <method name="isIncremental" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Options that can be specified when manually triggering a block report.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.client.BlockReportOptions -->
+ <!-- start class org.apache.hadoop.hdfs.client.BlockReportOptions.Factory -->
+ <class name="BlockReportOptions.Factory" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="Factory"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="setIncremental" return="org.apache.hadoop.hdfs.client.BlockReportOptions.Factory"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="incremental" type="boolean"/>
+ </method>
+ <method name="build" return="org.apache.hadoop.hdfs.client.BlockReportOptions"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.client.BlockReportOptions.Factory -->
+ <!-- start class org.apache.hadoop.hdfs.client.HdfsAdmin -->
+ <class name="HdfsAdmin" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HdfsAdmin" type="java.net.URI, org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Create a new HdfsAdmin client.
+
+ @param uri the unique URI of the HDFS file system to administer
+ @param conf configuration
+ @throws IOException in the event the file system could not be created]]>
+ </doc>
+ </constructor>
+ <method name="setQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="quota" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the namespace quota (count of files, directories, and sym links) for a
+ directory.
+
+ @param src the path to set the quota for
+ @param quota the value to set for the quota
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="clearQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Clear the namespace quota (count of files, directories and sym links) for a
+ directory.
+
+ @param src the path to clear the quota of
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="setSpaceQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="spaceQuota" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the storage space quota (size of files) for a directory. Note that
+ directories and sym links do not occupy storage space.
+
+ @param src the path to set the space quota of
+ @param spaceQuota the value to set for the space quota
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="clearSpaceQuota"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Clear the storage space quota (size of files) for a directory. Note that
+ directories and sym links do not occupy storage space.
+
+ @param src the path to clear the space quota of
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="setQuotaByStorageType"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+ <param name="quota" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the quota by storage type for a directory. Note that
+ directories and sym links do not occupy storage type quota.
+
+ @param src the target directory to set the quota by storage type
+ @param type the storage type to set for quota by storage type
+ @param quota the value to set for quota by storage type
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="clearQuotaByStorageType"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="type" type="org.apache.hadoop.fs.StorageType"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Clear the space quota by storage type for a directory. Note that
+ directories and sym links do not occupy storage type quota.
+
+ @param src the target directory to clear the quota by storage type
+ @param type the storage type to clear for quota by storage type
+ @throws IOException in the event of error]]>
+ </doc>
+ </method>
+ <method name="allowSnapshot"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Allow snapshot on a directory.
+ @param path The path of the directory where snapshots will be taken.]]>
+ </doc>
+ </method>
+ <method name="disallowSnapshot"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Disallow snapshot on a directory.
+ @param path The path of the snapshottable directory.]]>
+ </doc>
+ </method>
+ <method name="addCacheDirective" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo"/>
+ <param name="flags" type="java.util.EnumSet"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Add a new CacheDirectiveInfo.
+
+ @param info Information about a directive to add.
+ @param flags {@link CacheFlag}s to use for this operation.
+ @return the ID of the directive that was created.
+ @throws IOException if the directive could not be added]]>
+ </doc>
+ </method>
+ <method name="modifyCacheDirective"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo"/>
+ <param name="flags" type="java.util.EnumSet"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Modify a CacheDirective.
+
+ @param info Information about the directive to modify. You must set the ID
+ to indicate which CacheDirective you want to modify.
+ @param flags {@link CacheFlag}s to use for this operation.
+ @throws IOException if the directive could not be modified]]>
+ </doc>
+ </method>
+ <method name="removeCacheDirective"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="id" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Remove a CacheDirective.
+
+ @param id identifier of the CacheDirectiveInfo to remove
+ @throws IOException if the directive could not be removed]]>
+ </doc>
+ </method>
+ <method name="listCacheDirectives" return="org.apache.hadoop.fs.RemoteIterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="filter" type="org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[List cache directives. Incrementally fetches results from the server.
+
+ @param filter Filter parameters to use when listing the directives, null to
+ list all directives visible to us.
+ @return A RemoteIterator which returns CacheDirectiveInfo objects.]]>
+ </doc>
+ </method>
+ <method name="addCachePool"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CachePoolInfo"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Add a cache pool.
+
+ @param info
+ The request to add a cache pool.
+ @throws IOException
+ If the request could not be completed.]]>
+ </doc>
+ </method>
+ <method name="modifyCachePool"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="info" type="org.apache.hadoop.hdfs.protocol.CachePoolInfo"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Modify an existing cache pool.
+
+ @param info
+ The request to modify a cache pool.
+ @throws IOException
+ If the request could not be completed.]]>
+ </doc>
+ </method>
+ <method name="removeCachePool"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="poolName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Remove a cache pool.
+
+ @param poolName
+ Name of the cache pool to remove.
+ @throws IOException
+ if the cache pool did not exist, or could not be removed.]]>
+ </doc>
+ </method>
+ <method name="listCachePools" return="org.apache.hadoop.fs.RemoteIterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[List all cache pools.
+
+ @return A remote iterator from which you can get CachePoolEntry objects.
+ Requests will be made as needed.
+ @throws IOException
+ If there was an error listing cache pools.]]>
+ </doc>
+ </method>
+ <method name="createEncryptionZone"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <param name="keyName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+ <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+ <doc>
+ <![CDATA[Create an encryption zone rooted at an empty existing directory, using the
+ specified encryption key. An encryption zone has an associated encryption
+ key used when reading and writing files within the zone.
+
+ @param path The path of the root of the encryption zone. Must refer to
+ an empty, existing directory.
+ @param keyName Name of key available at the KeyProvider.
+ @throws IOException if there was a general IO exception
+ @throws AccessControlException if the caller does not have access to path
+ @throws FileNotFoundException if the path does not exist]]>
+ </doc>
+ </method>
+ <method name="getEncryptionZoneForPath" return="org.apache.hadoop.hdfs.protocol.EncryptionZone"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="path" type="org.apache.hadoop.fs.Path"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <exception name="AccessControlException" type="org.apache.hadoop.security.AccessControlException"/>
+ <exception name="FileNotFoundException" type="java.io.FileNotFoundException"/>
+ <doc>
+ <![CDATA[Get the path of the encryption zone for a given file or directory.
+
+ @param path The path to get the ez for.
+
+ @return The EncryptionZone of the ez, or null if path is not in an ez.
+ @throws IOException if there was a general IO exception
+ @throws AccessControlException if the caller does not have access to path
+ @throws FileNotFoundException if the path does not exist]]>
+ </doc>
+ </method>
+ <method name="listEncryptionZones" return="org.apache.hadoop.fs.RemoteIterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Returns a RemoteIterator which can be used to list the encryption zones
+ in HDFS. For large numbers of encryption zones, the iterator will fetch
+ the list of zones in a number of small batches.
+ <p/>
+ Since the list is fetched in batches, it does not represent a
+ consistent snapshot of the entire list of encryption zones.
+ <p/>
+ This method can only be called by HDFS superusers.]]>
+ </doc>
+ </method>
+ <method name="getInotifyEventStream" return="org.apache.hadoop.hdfs.DFSInotifyEventInputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Exposes a stream of namesystem events. Only events occurring after the
+ stream is created are available.
+ See {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream}
+ for information on stream usage.
+ See {@link org.apache.hadoop.hdfs.inotify.Event}
+ for information on the available events.
+ <p/>
+ Inotify users may want to tune the following HDFS parameters to
+ ensure that enough extra HDFS edits are saved to support inotify clients
+ that fall behind the current state of the namespace while reading events.
+ The default parameter values should generally be reasonable. If edits are
+ deleted before their corresponding events can be read, clients will see a
+ {@link org.apache.hadoop.hdfs.inotify.MissingEventsException} on
+ {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream} method calls.
+
+ It should generally be sufficient to tune these parameters:
+ dfs.namenode.num.extra.edits.retained
+ dfs.namenode.max.extra.edits.segments.retained
+
+ Parameters that affect the number of created segments and the number of
+ edits that are considered necessary, i.e. do not count towards the
+ dfs.namenode.num.extra.edits.retained quota):
+ dfs.namenode.checkpoint.period
+ dfs.namenode.checkpoint.txns
+ dfs.namenode.num.checkpoints.retained
+ dfs.ha.log-roll.period
+ <p/>
+ It is recommended that local journaling be configured
+ (dfs.namenode.edits.dir) for inotify (in addition to a shared journal)
+ so that edit transfers from the shared journal can be avoided.
+
+ @throws IOException If there was an error obtaining the stream.]]>
+ </doc>
+ </method>
+ <method name="getInotifyEventStream" return="org.apache.hadoop.hdfs.DFSInotifyEventInputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="lastReadTxid" type="long"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[A version of {@link HdfsAdmin#getInotifyEventStream()} meant for advanced
+ users who are aware of HDFS edits up to lastReadTxid (e.g. because they
+ have access to an FSImage inclusive of lastReadTxid) and only want to read
+ events after this point.]]>
+ </doc>
+ </method>
+ <method name="setStoragePolicy"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="src" type="org.apache.hadoop.fs.Path"/>
+ <param name="policyName" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Set the source path to the specified storage policy.
+
+ @param src The source path referring to either a directory or a file.
+ @param policyName The name of the storage policy.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[The public API for performing administrative functions on HDFS. Those writing
+ applications against HDFS should prefer this interface to directly accessing
+ functionality in DistributedFileSystem or DFSClient.
+
+ Note that this is distinct from the similarly-named {@link DFSAdmin}, which
+ is a class that provides the functionality for the CLI `hdfs dfsadmin ...'
+ commands.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.hdfs.client.HdfsAdmin -->
+ <!-- start class org.apache.hadoop.hdfs.client.HdfsDataInputStream -->
+ <class name="HdfsDataInputStream" extends="org.apache.hadoop.fs.FSDataInputStream"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HdfsDataInputStream" type="org.apache.hadoop.hdfs.DFSInputStream"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <constructor name="HdfsDataInputStream" type="org.apache.hadoop.crypto.CryptoInputStream"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ </constructor>
+ <method name="getWrappedStream" return="java.io.InputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get a reference to the wrapped output stream. We always want to return the
+ actual underlying InputStream, even when we're using a CryptoStream. e.g.
+ in the delegated methods below.
+
+ @return the underlying output stream]]>
+ </doc>
+ </method>
+ <method name="getCurrentDatanode" return="org.apache.hadoop.hdfs.protocol.DatanodeInfo"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the datanode from which the stream is currently reading.]]>
+ </doc>
+ </method>
+ <method name="getCurrentBlock" return="org.apache.hadoop.hdfs.protocol.ExtendedBlock"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the block containing the target position.]]>
+ </doc>
+ </method>
+ <method name="getAllBlocks" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the collection of blocks that has already been located.]]>
+ </doc>
+ </method>
+ <method name="getVisibleLength" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the visible length of the file. It will
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/52] [abbrv] hadoop git commit: YARN-5557. Add localize API to the
ContainerManagementProtocol. Contributed by Jian He.
Posted by aw...@apache.org.
YARN-5557. Add localize API to the ContainerManagementProtocol. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ef632f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ef632f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ef632f3
Branch: refs/heads/HADOOP-13341
Commit: 9ef632f3b0b0e0808116cd1c7482a205b7ebef7d
Parents: 13fb1b5
Author: Junping Du <ju...@apache.org>
Authored: Fri Aug 26 09:03:44 2016 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Fri Aug 26 09:04:44 2016 -0700
----------------------------------------------------------------------
.../v2/app/launcher/TestContainerLauncher.java | 8 +
.../app/launcher/TestContainerLauncherImpl.java | 8 +
.../yarn/api/ContainerManagementProtocol.java | 16 ++
.../ResourceLocalizationRequest.java | 84 ++++++++
.../ResourceLocalizationResponse.java | 40 ++++
.../proto/containermanagement_protocol.proto | 1 +
.../src/main/proto/yarn_service_protos.proto | 7 +
...ContainerManagementProtocolPBClientImpl.java | 38 +++-
...ontainerManagementProtocolPBServiceImpl.java | 20 ++
.../pb/ResourceLocalizationRequestPBImpl.java | 216 +++++++++++++++++++
.../pb/ResourceLocalizationResponsePBImpl.java | 69 ++++++
.../hadoop/yarn/TestContainerLaunchRPC.java | 8 +
.../yarn/TestContainerResourceIncreaseRPC.java | 7 +
.../hadoop/yarn/api/TestPBImplRecords.java | 15 ++
.../java/org/apache/hadoop/yarn/TestRPC.java | 8 +
.../containermanager/ContainerManagerImpl.java | 10 +
.../nodemanager/DummyContainerManager.java | 8 +
.../server/resourcemanager/NodeManager.java | 8 +
.../resourcemanager/TestAMAuthorization.java | 8 +
.../TestApplicationMasterLauncher.java | 8 +
20 files changed, 577 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
index f1c5b77..ba404a5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
@@ -32,6 +32,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -468,5 +470,11 @@ public class TestContainerLauncher {
SignalContainerRequest request) throws YarnException, IOException {
return null;
}
+
+ @Override
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
index d04f08c..be1cad9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -50,6 +50,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequ
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -473,6 +475,12 @@ public class TestContainerLauncherImpl {
SignalContainerRequest request) throws YarnException, IOException {
return null;
}
+
+ @Override
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ return null;
+ }
}
@SuppressWarnings("serial")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
index c6c0de5..f8f03b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequ
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -199,4 +201,18 @@ public interface ContainerManagementProtocol {
SignalContainerResponse signalToContainer(SignalContainerRequest request)
throws YarnException, IOException;
+
+ /**
+ * Localize resources required by the container.
+ * Currently, this API only works for running containers.
+ *
+ * @param request Specify the resources to be localized.
+ * @return Response that the localize request is accepted.
+ * @throws YarnException Exception specific to YARN
+ * @throws IOException IOException thrown from the RPC layer.
+ */
+ @Public
+ @Unstable
+ ResourceLocalizationResponse localize(ResourceLocalizationRequest request)
+ throws YarnException, IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceLocalizationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceLocalizationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceLocalizationRequest.java
new file mode 100644
index 0000000..2062de7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceLocalizationRequest.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.util.Map;
+
+/**
+ * The request sent by the ApplicationMaster to ask for localizing resources.
+ */
+@Public
+@Unstable
+public abstract class ResourceLocalizationRequest {
+
+ @Public
+ @Unstable
+ public static ResourceLocalizationRequest newInstance(ContainerId containerId,
+ Map<String, LocalResource> localResources) {
+ ResourceLocalizationRequest record =
+ Records.newRecord(ResourceLocalizationRequest.class);
+ record.setContainerId(containerId);
+ record.setLocalResources(localResources);
+ return record;
+ }
+
+ /**
+ * Get the <code>ContainerId</code> of the container to localize resources.
+ *
+ * @return <code>ContainerId</code> of the container to localize resources.
+ */
+ @Public
+ @Unstable
+ public abstract ContainerId getContainerId();
+
+ /**
+ * Set the <code>ContainerId</code> of the container to localize resources.
+ * @param containerId the containerId of the container.
+ */
+ @Private
+ @Unstable
+ public abstract void setContainerId(ContainerId containerId);
+
+ /**
+ * Get <code>LocalResource</code> required by the container.
+ *
+ * @return all <code>LocalResource</code> required by the container
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, LocalResource> getLocalResources();
+
+ /**
+ * Set <code>LocalResource</code> required by the container. All pre-existing
+ * Map entries are cleared before adding the new Map
+ *
+ * @param localResources <code>LocalResource</code> required by the container
+ */
+ @Private
+ @Unstable
+ public abstract void setLocalResources(
+ Map<String, LocalResource> localResources);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceLocalizationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceLocalizationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceLocalizationResponse.java
new file mode 100644
index 0000000..6ebf0a5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/ResourceLocalizationResponse.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * The response to the {@link ResourceLocalizationRequest}
+ */
+@Public
+@Unstable
+public abstract class ResourceLocalizationResponse {
+
+ @Private
+ @Unstable
+ public static ResourceLocalizationResponse newInstance() {
+ ResourceLocalizationResponse record =
+ Records.newRecord(ResourceLocalizationResponse.class);
+ return record;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
index fc00505..a28fa27 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto
@@ -36,4 +36,5 @@ service ContainerManagementProtocolService {
rpc getContainerStatuses(GetContainerStatusesRequestProto) returns (GetContainerStatusesResponseProto);
rpc increaseContainersResource(IncreaseContainersResourceRequestProto) returns (IncreaseContainersResourceResponseProto);
rpc signalToContainer(SignalContainerRequestProto) returns (SignalContainerResponseProto);
+ rpc localize(ResourceLocalizationRequestProto) returns (ResourceLocalizationResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index dddb0f0..4abb80b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -263,6 +263,13 @@ message StopContainerRequestProto {
message StopContainerResponseProto {
}
+message ResourceLocalizationRequestProto {
+ optional ContainerIdProto container_id = 1;
+ repeated StringLocalResourceMapProto local_resources = 2;
+}
+
+message ResourceLocalizationResponseProto {
+}
//// bulk API records
message StartContainersRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java
index dfc672e..2c4fade 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java
@@ -18,10 +18,7 @@
package org.apache.hadoop.yarn.api.impl.pb.client;
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
+import com.google.protobuf.ServiceException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -30,20 +27,24 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
-import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceRequestPBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ResourceLocalizationRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ResourceLocalizationResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl;
@@ -54,12 +55,15 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ResourceLocalizationRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SignalContainerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProto;
-import com.google.protobuf.ServiceException;
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
@Private
public class ContainerManagementProtocolPBClientImpl implements ContainerManagementProtocol,
@@ -167,4 +171,18 @@ public class ContainerManagementProtocolPBClientImpl implements ContainerManagem
return null;
}
}
+
+ @Override
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ ResourceLocalizationRequestProto requestProto =
+ ((ResourceLocalizationRequestPBImpl) request).getProto();
+ try {
+ return new ResourceLocalizationResponsePBImpl(
+ proxy.localize(null, requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java
index 1744f33..9e34d66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
@@ -32,6 +33,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersReso
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ResourceLocalizationRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ResourceLocalizationResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SignalContainerResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl;
@@ -49,6 +52,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProt
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ResourceLocalizationRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ResourceLocalizationResponseProto;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -136,4 +141,19 @@ public class ContainerManagementProtocolPBServiceImpl implements ContainerManage
throw new ServiceException(e);
}
}
+
+ @Override
+ public ResourceLocalizationResponseProto localize(RpcController controller,
+ ResourceLocalizationRequestProto proto) throws ServiceException {
+ ResourceLocalizationRequestPBImpl request =
+ new ResourceLocalizationRequestPBImpl(proto);
+ try {
+ ResourceLocalizationResponse response = real.localize(request);
+ return ((ResourceLocalizationResponsePBImpl) response).getProto();
+ } catch (YarnException e) {
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ResourceLocalizationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ResourceLocalizationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ResourceLocalizationRequestPBImpl.java
new file mode 100644
index 0000000..2ca6fef
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ResourceLocalizationRequestPBImpl.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import com.google.protobuf.TextFormat;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ResourceLocalizationRequestProto;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public class ResourceLocalizationRequestPBImpl
+ extends ResourceLocalizationRequest {
+ private ResourceLocalizationRequestProto proto =
+ ResourceLocalizationRequestProto.getDefaultInstance();
+ private ResourceLocalizationRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private Map<String, LocalResource> localResources = null;
+ private ContainerId containerId;
+
+ public ResourceLocalizationRequestPBImpl() {
+ builder = ResourceLocalizationRequestProto.newBuilder();
+ }
+
+ public ResourceLocalizationRequestPBImpl(
+ ResourceLocalizationRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.containerId != null) {
+ builder.setContainerId(convertToProtoFormat(this.containerId));
+ }
+ if (this.localResources != null) {
+ addLocalResourcesToProto();
+ }
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ResourceLocalizationRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void addLocalResourcesToProto() {
+ maybeInitBuilder();
+ builder.clearLocalResources();
+ if (localResources == null) {
+ return;
+ }
+ Iterable<YarnProtos.StringLocalResourceMapProto> iterable =
+ new Iterable<YarnProtos.StringLocalResourceMapProto>() {
+
+ @Override
+ public Iterator<YarnProtos.StringLocalResourceMapProto> iterator() {
+ return new Iterator<YarnProtos.StringLocalResourceMapProto>() {
+
+ Iterator<String> keyIter = localResources.keySet().iterator();
+
+ @Override public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override public YarnProtos.StringLocalResourceMapProto next() {
+ String key = keyIter.next();
+ return YarnProtos.StringLocalResourceMapProto.newBuilder()
+ .setKey(key).
+ setValue(convertToProtoFormat(localResources.get(key)))
+ .build();
+ }
+
+ @Override public boolean hasNext() {
+ return keyIter.hasNext();
+ }
+ };
+ }
+ };
+ builder.addAllLocalResources(iterable);
+ }
+
+ public ResourceLocalizationRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void initLocalResources() {
+ if (this.localResources != null) {
+ return;
+ }
+ YarnServiceProtos.ResourceLocalizationRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List<YarnProtos.StringLocalResourceMapProto> list =
+ p.getLocalResourcesList();
+ this.localResources = new HashMap<>();
+
+ for (YarnProtos.StringLocalResourceMapProto c : list) {
+ this.localResources.put(c.getKey(), convertFromProtoFormat(c.getValue()));
+ }
+ }
+
+ private LocalResourcePBImpl convertFromProtoFormat(
+ YarnProtos.LocalResourceProto p) {
+ return new LocalResourcePBImpl(p);
+ }
+
+ private ContainerIdPBImpl convertFromProtoFormat(
+ YarnProtos.ContainerIdProto p) {
+ return new ContainerIdPBImpl(p);
+ }
+
+ private YarnProtos.ContainerIdProto convertToProtoFormat(ContainerId t) {
+ return ((ContainerIdPBImpl) t).getProto();
+ }
+
+ private YarnProtos.LocalResourceProto convertToProtoFormat(LocalResource t) {
+ return ((LocalResourcePBImpl) t).getProto();
+ }
+
+ @Override
+ public ContainerId getContainerId() {
+ YarnServiceProtos.ResourceLocalizationRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (this.containerId != null) {
+ return this.containerId;
+ }
+ if (!p.hasContainerId()) {
+ return null;
+ }
+ this.containerId = convertFromProtoFormat(p.getContainerId());
+ return this.containerId;
+ }
+
+ @Override
+ public void setContainerId(ContainerId containerId) {
+ maybeInitBuilder();
+ if (containerId == null) {
+ builder.clearContainerId();
+ }
+ this.containerId = containerId;
+ }
+
+ @Override
+ public Map<String, LocalResource> getLocalResources() {
+ initLocalResources();
+ return this.localResources;
+ }
+
+ @Override
+ public void setLocalResources(Map<String, LocalResource> localResources) {
+ if (localResources == null) {
+ this.localResources = null;
+ builder.clearLocalResources();
+ return;
+ }
+ this.localResources = new HashMap<>(localResources);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ResourceLocalizationResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ResourceLocalizationResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ResourceLocalizationResponsePBImpl.java
new file mode 100644
index 0000000..4d6ae7c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/ResourceLocalizationResponsePBImpl.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+
+import com.google.protobuf.TextFormat;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.ResourceLocalizationResponseProto;
+
+public class ResourceLocalizationResponsePBImpl extends
+ ResourceLocalizationResponse {
+ ResourceLocalizationResponseProto proto =
+ ResourceLocalizationResponseProto.getDefaultInstance();
+ ResourceLocalizationResponseProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public YarnServiceProtos.ResourceLocalizationResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ public ResourceLocalizationResponsePBImpl() {
+ builder = ResourceLocalizationResponseProto.newBuilder();
+ }
+
+ public ResourceLocalizationResponsePBImpl(
+ ResourceLocalizationResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
index 31bea2d..8dcf029 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
@@ -36,6 +36,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequ
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -198,5 +200,11 @@ public class TestContainerLaunchRPC {
"Dummy function cause"));
throw new YarnException(e);
}
+
+ @Override
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
index f366285..b6ae283 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -180,5 +182,10 @@ public class TestContainerResourceIncreaseRPC {
SignalContainerRequest request) throws YarnException, IOException {
return null;
}
+
+ @Override public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 538b25a..da52f14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -100,6 +100,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationSubmissionR
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationSubmissionResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReservationUpdateResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ResourceLocalizationRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ResourceLocalizationResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersResponsePBImpl;
@@ -246,6 +248,7 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.Repla
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto;
@@ -985,6 +988,18 @@ public class TestPBImplRecords {
}
@Test
+ public void testResourceLocalizationRequest() throws Exception {
+ validatePBImplRecord(ResourceLocalizationRequestPBImpl.class,
+ YarnServiceProtos.ResourceLocalizationRequestProto.class);
+ }
+
+ @Test
+ public void testResourceLocalizationResponse() throws Exception {
+ validatePBImplRecord(ResourceLocalizationResponsePBImpl.class,
+ YarnServiceProtos.ResourceLocalizationResponseProto.class);
+ }
+
+ @Test
public void testContainerPBImpl() throws Exception {
validatePBImplRecord(ContainerPBImpl.class, ContainerProto.class);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
index e25f528..f54a270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResp
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -347,6 +349,12 @@ public class TestRPC {
new Exception(EXCEPTION_CAUSE));
throw new YarnException(e);
}
+
+ @Override
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ return null;
+ }
}
public static ContainerTokenIdentifier newContainerTokenIdentifier(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index cb63ae3..de79e89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -59,6 +59,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -1514,6 +1516,14 @@ public class ContainerManagerImpl extends CompositeService implements
return new SignalContainerResponsePBImpl();
}
+ @Override
+ @SuppressWarnings("unchecked")
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+
+ return ResourceLocalizationResponse.newInstance();
+ }
+
@SuppressWarnings("unchecked")
private void internalSignalToContainer(SignalContainerRequest request,
String sentBy) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
index 3ff04d8..4aeb2e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.util.Collection;
import org.apache.commons.logging.Log;
@@ -27,6 +28,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -204,4 +207,9 @@ public class DummyContainerManager extends ContainerManagerImpl {
// do nothing
}
+ @Override
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ return null;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
index 5afec84..4416010 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@@ -27,6 +27,8 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -324,4 +326,10 @@ public class NodeManager implements ContainerManagementProtocol {
SignalContainerRequest request) throws YarnException, IOException {
throw new YarnException("Not supported yet!");
}
+
+ @Override
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
index 0f88c79..f325cd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
@@ -47,6 +47,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -168,6 +170,12 @@ public class TestAMAuthorization {
SignalContainerRequest request) throws YarnException, IOException {
return null;
}
+
+ @Override
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ return null;
+ }
}
public static class MockRMWithAMS extends MockRMWithCustomAMLauncher {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ef632f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
index 3482af2..15b8ade 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequ
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -156,6 +158,12 @@ public class TestApplicationMasterLauncher {
SignalContainerRequest request) throws YarnException, IOException {
return null;
}
+
+ @Override
+ public ResourceLocalizationResponse localize(
+ ResourceLocalizationRequest request) throws YarnException, IOException {
+ return null;
+ }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/52] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by aw...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
deleted file mode 100644
index a22dd28..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.s3a.yarn;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FsStatus;
-import org.apache.hadoop.fs.Path;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import java.util.EnumSet;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-public class TestS3A {
- private FileContext fc;
-
- @Rule
- public final Timeout testTimeout = new Timeout(90000);
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- fc = S3ATestUtils.createTestFileContext(conf);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fc != null) {
- fc.delete(getTestPath(), true);
- }
- }
-
- protected Path getTestPath() {
- return new Path("/tests3afc");
- }
-
- @Test
- public void testS3AStatus() throws Exception {
- FsStatus fsStatus = fc.getFsStatus(null);
- assertNotNull(fsStatus);
- assertTrue("Used capacity should be positive: " + fsStatus.getUsed(),
- fsStatus.getUsed() >= 0);
- assertTrue("Remaining capacity should be positive: " + fsStatus
- .getRemaining(),
- fsStatus.getRemaining() >= 0);
- assertTrue("Capacity should be positive: " + fsStatus.getCapacity(),
- fsStatus.getCapacity() >= 0);
- }
-
- @Test
- public void testS3ACreateFileInSubDir() throws Exception {
- Path dirPath = getTestPath();
- fc.mkdir(dirPath,FileContext.DIR_DEFAULT_PERM,true);
- Path filePath = new Path(dirPath, "file");
- try (FSDataOutputStream file = fc.create(filePath, EnumSet.of(CreateFlag
- .CREATE))) {
- file.write(666);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java
deleted file mode 100644
index 990d79f..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.fs.s3a.yarn;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.examples.WordCount;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3ATestUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-
-import org.junit.After;
-import static org.junit.Assert.assertTrue;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
-public class TestS3AMiniYarnCluster {
-
- private final Configuration conf = new YarnConfiguration();
- private S3AFileSystem fs;
- private MiniYARNCluster yarnCluster;
- private final String rootPath = "/tests/MiniClusterWordCount/";
-
- @Before
- public void beforeTest() throws IOException {
- fs = S3ATestUtils.createTestFileSystem(conf);
- fs.mkdirs(new Path(rootPath + "input/"));
-
- yarnCluster = new MiniYARNCluster("MiniClusterWordCount", // testName
- 1, // number of node managers
- 1, // number of local log dirs per node manager
- 1); // number of hdfs dirs per node manager
- yarnCluster.init(conf);
- yarnCluster.start();
- }
-
- @After
- public void afterTest() throws IOException {
- fs.delete(new Path(rootPath), true);
- yarnCluster.stop();
- }
-
- @Test
- public void testWithMiniCluster() throws Exception {
- Path input = new Path(rootPath + "input/in.txt");
- input = input.makeQualified(fs.getUri(), fs.getWorkingDirectory());
- Path output = new Path(rootPath + "output/");
- output = output.makeQualified(fs.getUri(), fs.getWorkingDirectory());
-
- writeStringToFile(input, "first line\nsecond line\nthird line");
-
- Job job = Job.getInstance(conf, "word count");
- job.setJarByClass(WordCount.class);
- job.setMapperClass(WordCount.TokenizerMapper.class);
- job.setCombinerClass(WordCount.IntSumReducer.class);
- job.setReducerClass(WordCount.IntSumReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- FileInputFormat.addInputPath(job, input);
- FileOutputFormat.setOutputPath(job, output);
-
- int exitCode = (job.waitForCompletion(true) ? 0 : 1);
- assertEquals("Returned error code.", 0, exitCode);
-
- assertTrue(fs.exists(new Path(output, "_SUCCESS")));
- String outputAsStr = readStringFromFile(new Path(output, "part-r-00000"));
- Map<String, Integer> resAsMap = getResultAsMap(outputAsStr);
-
- assertEquals(4, resAsMap.size());
- assertEquals(1, (int) resAsMap.get("first"));
- assertEquals(1, (int) resAsMap.get("second"));
- assertEquals(1, (int) resAsMap.get("third"));
- assertEquals(3, (int) resAsMap.get("line"));
- }
-
- /**
- * helper method
- */
- private Map<String, Integer> getResultAsMap(String outputAsStr) throws IOException {
- Map<String, Integer> result = new HashMap<>();
- for (String line : outputAsStr.split("\n")) {
- String[] tokens = line.split("\t");
- result.put(tokens[0], Integer.parseInt(tokens[1]));
- }
- return result;
- }
-
- /**
- * helper method
- */
- private void writeStringToFile(Path path, String string) throws IOException {
- FileContext fc = S3ATestUtils.createTestFileContext(conf);
- try (FSDataOutputStream file = fc.create(path,
- EnumSet.of(CreateFlag.CREATE))) {
- file.write(string.getBytes());
- }
- }
-
- /**
- * helper method
- */
- private String readStringFromFile(Path path) {
- try (FSDataInputStream in = fs.open(path)) {
- long bytesLen = fs.getFileStatus(path).getLen();
- byte[] buffer = new byte[(int) bytesLen];
- IOUtils.readFully(in, buffer, 0, buffer.length);
- return new String(buffer);
- } catch (IOException e) {
- throw new RuntimeException("Failed to read from [" + path + "]", e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestInMemoryNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestInMemoryNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestInMemoryNativeS3FileSystemContract.java
new file mode 100644
index 0000000..adbf950
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestInMemoryNativeS3FileSystemContract.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.IOException;
+
+/**
+ * S3N basic contract tests through mock in-memory S3 implementation.
+ */
+public class ITestInMemoryNativeS3FileSystemContract
+ extends NativeS3FileSystemContractBaseTest {
+
+ @Override
+ NativeFileSystemStore getNativeFileSystemStore() throws IOException {
+ return new InMemoryNativeFileSystemStore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeFileSystemStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeFileSystemStore.java
new file mode 100644
index 0000000..cfe622c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeFileSystemStore.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import static org.junit.Assert.*;
+import static org.junit.Assume.*;
+
+import org.junit.Before;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * S3N tests through live S3 service.
+ */
+public class ITestJets3tNativeFileSystemStore {
+ private Configuration conf;
+ private Jets3tNativeFileSystemStore store;
+ private NativeS3FileSystem fs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ store = new Jets3tNativeFileSystemStore();
+ fs = new NativeS3FileSystem(store);
+ conf.setBoolean("fs.s3n.multipart.uploads.enabled", true);
+ conf.setLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024);
+ fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ store.purge("test");
+ } catch (Exception e) {}
+ }
+
+ @BeforeClass
+ public static void checkSettings() throws Exception {
+ Configuration conf = new Configuration();
+ assumeNotNull(conf.get("fs.s3n.awsAccessKeyId"));
+ assumeNotNull(conf.get("fs.s3n.awsSecretAccessKey"));
+ assumeNotNull(conf.get("test.fs.s3n.name"));
+ }
+
+ protected void writeRenameReadCompare(Path path, long len)
+ throws IOException, NoSuchAlgorithmException {
+ // If len > fs.s3n.multipart.uploads.block.size,
+ // we'll use a multipart upload copy
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ OutputStream out = new BufferedOutputStream(
+ new DigestOutputStream(fs.create(path, false), digest));
+ for (long i = 0; i < len; i++) {
+ out.write('Q');
+ }
+ out.flush();
+ out.close();
+
+ assertTrue("Exists", fs.exists(path));
+
+ // Depending on if this file is over 5 GB or not,
+ // rename will cause a multipart upload copy
+ Path copyPath = path.suffix(".copy");
+ fs.rename(path, copyPath);
+
+ assertTrue("Copy exists", fs.exists(copyPath));
+
+ // Download file from S3 and compare the digest against the original
+ MessageDigest digest2 = MessageDigest.getInstance("MD5");
+ InputStream in = new BufferedInputStream(
+ new DigestInputStream(fs.open(copyPath), digest2));
+ long copyLen = 0;
+ while (in.read() != -1) {
+ copyLen++;
+ }
+ in.close();
+
+ assertEquals("Copy length matches original", len, copyLen);
+ assertArrayEquals("Digests match", digest.digest(), digest2.digest());
+ }
+
+ @Test
+ public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
+ // Regular upload, regular copy
+ writeRenameReadCompare(new Path("/test/small"), 16384);
+ }
+
+ @Test
+ public void testMediumUpload() throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, regular copy
+ writeRenameReadCompare(new Path("/test/medium"), 33554432); // 100 MB
+ }
+
+ /*
+ Enable Multipart upload to run this test
+ @Test
+ public void testExtraLargeUpload()
+ throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, multipart copy
+ writeRenameReadCompare(new Path("/test/xlarge"), 5368709121L); // 5GB+1byte
+ }
+ */
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeS3FileSystemContract.java
new file mode 100644
index 0000000..e51eaf6
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/ITestJets3tNativeS3FileSystemContract.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3native;
+
+import java.io.IOException;
+
+/**
+ * S3N basic contract tests through live S3 service.
+ */
+public class ITestJets3tNativeS3FileSystemContract
+ extends NativeS3FileSystemContractBaseTest {
+
+ @Override
+ NativeFileSystemStore getNativeFileSystemStore() throws IOException {
+ return new Jets3tNativeFileSystemStore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
deleted file mode 100644
index 664d39e..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestInMemoryNativeS3FileSystemContract.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.IOException;
-
-public class TestInMemoryNativeS3FileSystemContract
- extends NativeS3FileSystemContractBaseTest {
-
- @Override
- NativeFileSystemStore getNativeFileSystemStore() throws IOException {
- return new InMemoryNativeFileSystemStore();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
deleted file mode 100644
index dbd476e..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeFileSystemStore.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import static org.junit.Assert.*;
-import static org.junit.Assume.*;
-
-import org.junit.Before;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.security.DigestInputStream;
-import java.security.DigestOutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-
-public class TestJets3tNativeFileSystemStore {
- private Configuration conf;
- private Jets3tNativeFileSystemStore store;
- private NativeS3FileSystem fs;
-
- @Before
- public void setUp() throws Exception {
- conf = new Configuration();
- store = new Jets3tNativeFileSystemStore();
- fs = new NativeS3FileSystem(store);
- conf.setBoolean("fs.s3n.multipart.uploads.enabled", true);
- conf.setLong("fs.s3n.multipart.uploads.block.size", 64 * 1024 * 1024);
- fs.initialize(URI.create(conf.get("test.fs.s3n.name")), conf);
- }
-
- @After
- public void tearDown() throws Exception {
- try {
- store.purge("test");
- } catch (Exception e) {}
- }
-
- @BeforeClass
- public static void checkSettings() throws Exception {
- Configuration conf = new Configuration();
- assumeNotNull(conf.get("fs.s3n.awsAccessKeyId"));
- assumeNotNull(conf.get("fs.s3n.awsSecretAccessKey"));
- assumeNotNull(conf.get("test.fs.s3n.name"));
- }
-
- protected void writeRenameReadCompare(Path path, long len)
- throws IOException, NoSuchAlgorithmException {
- // If len > fs.s3n.multipart.uploads.block.size,
- // we'll use a multipart upload copy
- MessageDigest digest = MessageDigest.getInstance("MD5");
- OutputStream out = new BufferedOutputStream(
- new DigestOutputStream(fs.create(path, false), digest));
- for (long i = 0; i < len; i++) {
- out.write('Q');
- }
- out.flush();
- out.close();
-
- assertTrue("Exists", fs.exists(path));
-
- // Depending on if this file is over 5 GB or not,
- // rename will cause a multipart upload copy
- Path copyPath = path.suffix(".copy");
- fs.rename(path, copyPath);
-
- assertTrue("Copy exists", fs.exists(copyPath));
-
- // Download file from S3 and compare the digest against the original
- MessageDigest digest2 = MessageDigest.getInstance("MD5");
- InputStream in = new BufferedInputStream(
- new DigestInputStream(fs.open(copyPath), digest2));
- long copyLen = 0;
- while (in.read() != -1) {copyLen++;}
- in.close();
-
- assertEquals("Copy length matches original", len, copyLen);
- assertArrayEquals("Digests match", digest.digest(), digest2.digest());
- }
-
- @Test
- public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
- // Regular upload, regular copy
- writeRenameReadCompare(new Path("/test/small"), 16384);
- }
-
- @Test
- public void testMediumUpload() throws IOException, NoSuchAlgorithmException {
- // Multipart upload, regular copy
- writeRenameReadCompare(new Path("/test/medium"), 33554432); // 100 MB
- }
-
- /*
- Enable Multipart upload to run this test
- @Test
- public void testExtraLargeUpload()
- throws IOException, NoSuchAlgorithmException {
- // Multipart upload, multipart copy
- writeRenameReadCompare(new Path("/test/xlarge"), 5368709121L); // 5GB+1byte
- }
- */
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeS3FileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeS3FileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeS3FileSystemContract.java
deleted file mode 100644
index 42d6f06..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestJets3tNativeS3FileSystemContract.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3native;
-
-import java.io.IOException;
-
-public class TestJets3tNativeS3FileSystemContract
- extends NativeS3FileSystemContractBaseTest {
-
- @Override
- NativeFileSystemStore getNativeFileSystemStore() throws IOException {
- return new Jets3tNativeFileSystemStore();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/52] [abbrv] hadoop git commit: MAPREDUCE-6764. Teragen LOG
initialization bug. Contributed by Yufei Gu.
Posted by aw...@apache.org.
MAPREDUCE-6764. Teragen LOG initialization bug. Contributed by Yufei Gu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1360bd2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1360bd2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1360bd2d
Branch: refs/heads/HADOOP-13341
Commit: 1360bd2d545134b582e70f2add33a105710dc80b
Parents: d288a0b
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Aug 25 09:49:23 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Aug 25 09:50:12 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1360bd2d/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
index d7d751a..22fe344 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
@@ -66,7 +66,7 @@ import org.apache.hadoop.util.ToolRunner;
* <b>bin/hadoop jar hadoop-*-examples.jar teragen 10000000000 in-dir</b>
*/
public class TeraGen extends Configured implements Tool {
- private static final Log LOG = LogFactory.getLog(TeraSort.class);
+ private static final Log LOG = LogFactory.getLog(TeraGen.class);
public static enum Counters {CHECKSUM}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/52] [abbrv] hadoop git commit: HADOOP-13527. Add Spark to
CallerContext LimitedPrivate scope. (Contributed by Weiqing Yang)
Posted by aw...@apache.org.
HADOOP-13527. Add Spark to CallerContext LimitedPrivate scope. (Contributed by Weiqing Yang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/115ecb52
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/115ecb52
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/115ecb52
Branch: refs/heads/HADOOP-13341
Commit: 115ecb52a86b49aad3d058a6b4c1c7926b8b0a40
Parents: d37b45d
Author: Mingliang Liu <li...@apache.org>
Authored: Sun Aug 21 09:40:29 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Sun Aug 21 09:40:29 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/ipc/CallerContext.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/115ecb52/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index 3d21bfe..bdfa471 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -33,7 +33,7 @@ import java.util.Arrays;
* This class is immutable.
*/
@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "Hive", "MapReduce",
- "Pig", "YARN"})
+ "Pig", "Spark", "YARN"})
@InterfaceStability.Evolving
public final class CallerContext {
public static final Charset SIGNATURE_ENCODING = StandardCharsets.UTF_8;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[52/52] [abbrv] hadoop git commit: HDFS-10795. Fix an error in
ReaderStrategy#ByteBufferStrategy. Contributed by Sammi Chen
Posted by aw...@apache.org.
HDFS-10795. Fix an error in ReaderStrategy#ByteBufferStrategy. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4a21d3a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4a21d3a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4a21d3a
Branch: refs/heads/HADOOP-13341
Commit: f4a21d3abaa7c5a9f0a0d8417e81f7eaf3d1b29a
Parents: 81485db
Author: Kai Zheng <ka...@intel.com>
Authored: Sat Aug 27 10:54:25 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Sat Aug 27 10:54:25 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4a21d3a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java
index d75a8ef..c984c3b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java
@@ -181,7 +181,7 @@ class ByteBufferStrategy implements ReaderStrategy {
int length) throws IOException {
ByteBuffer tmpBuf = readBuf.duplicate();
tmpBuf.limit(tmpBuf.position() + length);
- int nRead = blockReader.read(readBuf.slice());
+ int nRead = blockReader.read(tmpBuf);
// Only when data are read, update the position
if (nRead > 0) {
readBuf.position(readBuf.position() + nRead);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/52] [abbrv] hadoop git commit: HDFS-10772. Reduce byte/string
conversions for get listing. Contributed by Daryn Sharp.
Posted by aw...@apache.org.
HDFS-10772. Reduce byte/string conversions for get listing. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1f32937
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1f32937
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1f32937
Branch: refs/heads/HADOOP-13341
Commit: a1f3293762dddb0ca953d1145f5b53d9086b25b8
Parents: 3476156
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Aug 24 15:21:08 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Wed Aug 24 15:21:08 2016 -0500
----------------------------------------------------------------------
.../server/namenode/FSDirStatAndListingOp.java | 26 +++++++++++---------
1 file changed, 15 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1f32937/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 8a9393e..5072d68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.QuotaUsage;
@@ -52,7 +53,6 @@ import static org.apache.hadoop.util.Time.now;
class FSDirStatAndListingOp {
static DirectoryListing getListingInt(FSDirectory fsd, final String srcArg,
byte[] startAfter, boolean needLocation) throws IOException {
- final String startAfterString = DFSUtil.bytes2String(startAfter);
String src = null;
final INodesInPath iip;
@@ -65,16 +65,20 @@ class FSDirStatAndListingOp {
iip = fsd.getINodesInPath(src, true);
}
- // Get file name when startAfter is an INodePath
- if (FSDirectory.isReservedName(startAfterString)) {
- try {
- String tmp = FSDirectory.resolvePath(startAfterString, fsd);
- byte[][] regularPath = INode.getPathComponents(tmp);
- startAfter = regularPath[regularPath.length - 1];
- } catch (IOException e) {
- // Possibly the inode is deleted
- throw new DirectoryListingStartAfterNotFoundException(
- "Can't find startAfter " + startAfterString);
+ // Get file name when startAfter is an INodePath. This is not the
+ // common case so avoid any unnecessary processing unless required.
+ if (startAfter.length > 0 && startAfter[0] == Path.SEPARATOR_CHAR) {
+ final String startAfterString = DFSUtil.bytes2String(startAfter);
+ if (FSDirectory.isReservedName(startAfterString)) {
+ try {
+ byte[][] components = INode.getPathComponents(startAfterString);
+ components = FSDirectory.resolveComponents(components, fsd);
+ startAfter = components[components.length - 1];
+ } catch (IOException e) {
+ // Possibly the inode is deleted
+ throw new DirectoryListingStartAfterNotFoundException(
+ "Can't find startAfter " + startAfterString);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/52] [abbrv] hadoop git commit: HADOOP-13446. Support running
isolated unit tests separate from AWS integration tests. Contributed by Chris
Nauroth.
Posted by aw...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java
new file mode 100644
index 0000000..b0b8a65
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.hadoop.util.StopWatch;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Basic unit test for S3A's blocking executor service.
+ */
+public class ITestBlockingThreadPoolExecutorService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ BlockingThreadPoolExecutorService.class);
+
+ private static final int NUM_ACTIVE_TASKS = 4;
+ private static final int NUM_WAITING_TASKS = 2;
+ private static final int TASK_SLEEP_MSEC = 100;
+ private static final int SHUTDOWN_WAIT_MSEC = 200;
+ private static final int SHUTDOWN_WAIT_TRIES = 5;
+ private static final int BLOCKING_THRESHOLD_MSEC = 50;
+
+ private static final Integer SOME_VALUE = 1337;
+
+ private static BlockingThreadPoolExecutorService tpe = null;
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ ensureDestroyed();
+ }
+
+ /**
+ * Basic test of running one trivial task.
+ */
+ @Test
+ public void testSubmitCallable() throws Exception {
+ ensureCreated();
+ ListenableFuture<Integer> f = tpe.submit(callableSleeper);
+ Integer v = f.get();
+ assertEquals(SOME_VALUE, v);
+ }
+
+ /**
+ * More involved test, including detecting blocking when at capacity.
+ */
+ @Test
+ public void testSubmitRunnable() throws Exception {
+ ensureCreated();
+ int totalTasks = NUM_ACTIVE_TASKS + NUM_WAITING_TASKS;
+ StopWatch stopWatch = new StopWatch().start();
+ for (int i = 0; i < totalTasks; i++) {
+ tpe.submit(sleeper);
+ assertDidntBlock(stopWatch);
+ }
+ tpe.submit(sleeper);
+ assertDidBlock(stopWatch);
+ }
+
+ @Test
+ public void testShutdown() throws Exception {
+ // Cover create / destroy, regardless of when this test case runs
+ ensureCreated();
+ ensureDestroyed();
+
+ // Cover create, execute, destroy, regardless of when test case runs
+ ensureCreated();
+ testSubmitRunnable();
+ ensureDestroyed();
+ }
+
+ // Helper functions, etc.
+
+ private void assertDidntBlock(StopWatch sw) {
+ try {
+ assertFalse("Non-blocking call took too long.",
+ sw.now(TimeUnit.MILLISECONDS) > BLOCKING_THRESHOLD_MSEC);
+ } finally {
+ sw.reset().start();
+ }
+ }
+
+ private void assertDidBlock(StopWatch sw) {
+ try {
+ if (sw.now(TimeUnit.MILLISECONDS) < BLOCKING_THRESHOLD_MSEC) {
+ throw new RuntimeException("Blocking call returned too fast.");
+ }
+ } finally {
+ sw.reset().start();
+ }
+ }
+
+ private Runnable sleeper = new Runnable() {
+ @Override
+ public void run() {
+ String name = Thread.currentThread().getName();
+ try {
+ Thread.sleep(TASK_SLEEP_MSEC);
+ } catch (InterruptedException e) {
+ LOG.info("Thread {} interrupted.", name);
+ Thread.currentThread().interrupt();
+ }
+ }
+ };
+
+ private Callable<Integer> callableSleeper = new Callable<Integer>() {
+ @Override
+ public Integer call() throws Exception {
+ sleeper.run();
+ return SOME_VALUE;
+ }
+ };
+
+ /**
+ * Helper function to create thread pool under test.
+ */
+ private static void ensureCreated() throws Exception {
+ if (tpe == null) {
+ LOG.debug("Creating thread pool");
+ tpe = new BlockingThreadPoolExecutorService(NUM_ACTIVE_TASKS,
+ NUM_WAITING_TASKS, 1, TimeUnit.SECONDS, "btpetest");
+ }
+ }
+
+ /**
+ * Helper function to terminate thread pool under test, asserting that
+ * shutdown -> terminate works as expected.
+ */
+ private static void ensureDestroyed() throws Exception {
+ if (tpe == null) {
+ return;
+ }
+ int shutdownTries = SHUTDOWN_WAIT_TRIES;
+
+ tpe.shutdown();
+ if (!tpe.isShutdown()) {
+ throw new RuntimeException("Shutdown had no effect.");
+ }
+
+ while (!tpe.awaitTermination(SHUTDOWN_WAIT_MSEC,
+ TimeUnit.MILLISECONDS)) {
+ LOG.info("Waiting for thread pool shutdown.");
+ if (shutdownTries-- <= 0) {
+ LOG.error("Failed to terminate thread pool gracefully.");
+ break;
+ }
+ }
+ if (!tpe.isTerminated()) {
+ tpe.shutdownNow();
+ if (!tpe.awaitTermination(SHUTDOWN_WAIT_MSEC,
+ TimeUnit.MILLISECONDS)) {
+ throw new RuntimeException(
+ "Failed to terminate thread pool in timely manner.");
+ }
+ }
+ tpe = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
new file mode 100644
index 0000000..cf8783c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.AccessDeniedException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSCredentialsProviderChain;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.junit.Assert.*;
+
+/**
+ * Tests for {@link Constants#AWS_CREDENTIALS_PROVIDER} logic.
+ *
+ */
+public class ITestS3AAWSCredentialsProvider {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AAWSCredentialsProvider.class);
+
+ @Rule
+ public Timeout testTimeout = new Timeout(1 * 60 * 1000);
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ /**
+ * Declare what exception to raise, and the text which must be found
+ * in it.
+ * @param exceptionClass class of exception
+ * @param text text in exception
+ */
+ private void expectException(Class<? extends Throwable> exceptionClass,
+ String text) {
+ exception.expect(exceptionClass);
+ exception.expectMessage(text);
+ }
+
+ @Test
+ public void testBadConfiguration() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER, "no.such.class");
+ try {
+ createFailingFS(conf);
+ } catch (IOException e) {
+ if (!(e.getCause() instanceof ClassNotFoundException)) {
+ LOG.error("Unexpected nested cause: {} in {}", e.getCause(), e, e);
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Create a filesystem, expect it to fail by raising an IOException.
+ * Raises an assertion exception if in fact the FS does get instantiated.
+ * @param conf configuration
+ * @throws IOException an expected exception.
+ */
+ private void createFailingFS(Configuration conf) throws IOException {
+ S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf);
+ fs.listStatus(new Path("/"));
+ fail("Expected exception - got " + fs);
+ }
+
+ static class BadCredentialsProvider implements AWSCredentialsProvider {
+
+ @SuppressWarnings("unused")
+ public BadCredentialsProvider(URI name, Configuration conf) {
+ }
+
+ @Override
+ public AWSCredentials getCredentials() {
+ return new BasicAWSCredentials("bad_key", "bad_secret");
+ }
+
+ @Override
+ public void refresh() {
+ }
+ }
+
+ @Test
+ public void testBadCredentials() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER, BadCredentialsProvider.class.getName());
+ try {
+ createFailingFS(conf);
+ } catch (AccessDeniedException e) {
+ // expected
+ }
+ }
+
+ static class GoodCredentialsProvider extends AWSCredentialsProviderChain {
+
+ @SuppressWarnings("unused")
+ public GoodCredentialsProvider(URI name, Configuration conf) {
+ super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY),
+ conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider());
+ }
+ }
+
+ @Test
+ public void testGoodProvider() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName());
+ S3ATestUtils.createTestFileSystem(conf);
+ }
+
+ @Test
+ public void testAnonymousProvider() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER,
+ AnonymousAWSCredentialsProvider.class.getName());
+ Path testFile = new Path(
+ conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+ S3ATestUtils.useCSVDataEndpoint(conf);
+ FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
+ assertNotNull(fs);
+ assertTrue(fs instanceof S3AFileSystem);
+ FileStatus stat = fs.getFileStatus(testFile);
+ assertNotNull(stat);
+ assertEquals(testFile, stat.getPath());
+ }
+
+ /**
+ * A credential provider whose constructor signature doesn't match.
+ */
+ static class ConstructorSignatureErrorProvider
+ implements AWSCredentialsProvider {
+
+ @SuppressWarnings("unused")
+ public ConstructorSignatureErrorProvider(String str) {
+ }
+
+ @Override
+ public AWSCredentials getCredentials() {
+ return null;
+ }
+
+ @Override
+ public void refresh() {
+ }
+ }
+
+ /**
+ * A credential provider whose constructor raises an NPE.
+ */
+ static class ConstructorFailureProvider
+ implements AWSCredentialsProvider {
+
+ @SuppressWarnings("unused")
+ public ConstructorFailureProvider() {
+ throw new NullPointerException("oops");
+ }
+
+ @Override
+ public AWSCredentials getCredentials() {
+ return null;
+ }
+
+ @Override
+ public void refresh() {
+ }
+ }
+
+ @Test
+ public void testProviderWrongClass() throws Exception {
+ expectProviderInstantiationFailure(this.getClass().getName(),
+ NOT_AWS_PROVIDER);
+ }
+
+ @Test
+ public void testProviderNotAClass() throws Exception {
+ expectProviderInstantiationFailure("NoSuchClass",
+ "ClassNotFoundException");
+ }
+
+ private void expectProviderInstantiationFailure(String option,
+ String expectedErrorText) throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER, option);
+ Path testFile = new Path(
+ conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+ expectException(IOException.class, expectedErrorText);
+ URI uri = testFile.toUri();
+ S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
+ }
+
+ @Test
+ public void testProviderConstructorError() throws Exception {
+ expectProviderInstantiationFailure(
+ ConstructorSignatureErrorProvider.class.getName(),
+ CONSTRUCTOR_EXCEPTION);
+ }
+
+ @Test
+ public void testProviderFailureError() throws Exception {
+ expectProviderInstantiationFailure(
+ ConstructorFailureProvider.class.getName(),
+ INSTANTIATION_EXCEPTION);
+ }
+
+ @Test
+ public void testInstantiationChain() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(AWS_CREDENTIALS_PROVIDER,
+ TemporaryAWSCredentialsProvider.NAME
+ + ", \t" + SimpleAWSCredentialsProvider.NAME
+ + " ,\n " + AnonymousAWSCredentialsProvider.NAME);
+ Path testFile = new Path(
+ conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+
+ URI uri = testFile.toUri();
+ S3AUtils.createAWSCredentialProviderSet(uri, conf, uri);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java
new file mode 100644
index 0000000..4444d0c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+/**
+ * Demonstrate that the threadpool blocks additional client requests if
+ * its queue is full (rather than throwing an exception) by initiating an
+ * upload consisting of 4 parts with 2 threads and 1 spot in the queue. The
+ * 4th part should not trigger an exception as it would with a
+ * non-blocking threadpool.
+ */
+public class ITestS3ABlockingThreadPool {
+
+ private Configuration conf;
+ private S3AFileSystem fs;
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ protected Path getTestPath() {
+ return new Path("/tests3a");
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024);
+ conf.setLong(Constants.MULTIPART_SIZE, 5 * 1024 * 1024);
+ conf.setInt(Constants.MAX_THREADS, 2);
+ conf.setInt(Constants.MAX_TOTAL_TASKS, 1);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(getTestPath(), true);
+ }
+ }
+
+ @Test
+ public void testRegularMultiPartUpload() throws Exception {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 *
+ 1024);
+ }
+
+ @Test
+ public void testFastMultiPartUpload() throws Exception {
+ conf.setBoolean(Constants.FAST_UPLOAD, true);
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 *
+ 1024);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java
new file mode 100644
index 0000000..9a6dae7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.fileStatsToString;
+
+/**
+ * S3A tests for configuring block size.
+ */
+public class ITestS3ABlocksize extends AbstractFSContractTestBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3ABlocksize.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testBlockSize() throws Exception {
+ FileSystem fs = getFileSystem();
+ long defaultBlockSize = fs.getDefaultBlockSize();
+ assertEquals("incorrect blocksize",
+ S3AFileSystem.DEFAULT_BLOCKSIZE, defaultBlockSize);
+ long newBlockSize = defaultBlockSize * 2;
+ fs.getConf().setLong(Constants.FS_S3A_BLOCK_SIZE, newBlockSize);
+
+ Path dir = path("testBlockSize");
+ Path file = new Path(dir, "file");
+ createFile(fs, file, true, dataset(1024, 'a', 'z' - 'a'));
+ FileStatus fileStatus = fs.getFileStatus(file);
+ assertEquals("Double default block size in stat(): " + fileStatus,
+ newBlockSize,
+ fileStatus.getBlockSize());
+
+ // check the listing & assert that the block size is picked up by
+ // this route too.
+ boolean found = false;
+ FileStatus[] listing = fs.listStatus(dir);
+ for (FileStatus stat : listing) {
+ LOG.info("entry: {}", stat);
+ if (file.equals(stat.getPath())) {
+ found = true;
+ assertEquals("Double default block size in ls(): " + stat,
+ newBlockSize,
+ stat.getBlockSize());
+ }
+ }
+ assertTrue("Did not find " + fileStatsToString(listing, ", "), found);
+ }
+
+ @Test
+ public void testRootFileStatusHasBlocksize() throws Throwable {
+ FileSystem fs = getFileSystem();
+ FileStatus status = fs.getFileStatus(new Path("/"));
+ assertTrue("Invalid root blocksize",
+ status.getBlockSize() >= 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
new file mode 100644
index 0000000..4e99339
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -0,0 +1,435 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.reflect.FieldUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3native.S3xLoginHelper;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.net.URI;
+
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.http.HttpStatus;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * S3A tests for configuration.
+ */
+public class ITestS3AConfiguration {
+ private static final String EXAMPLE_ID = "AKASOMEACCESSKEY";
+ private static final String EXAMPLE_KEY =
+ "RGV0cm9pdCBSZ/WQgY2xl/YW5lZCB1cAEXAMPLE";
+
+ private Configuration conf;
+ private S3AFileSystem fs;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AConfiguration.class);
+
+ private static final String TEST_ENDPOINT = "test.fs.s3a.endpoint";
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Rule
+ public final TemporaryFolder tempDir = new TemporaryFolder();
+
+ /**
+ * Test if custom endpoint is picked up.
+ * <p/>
+ * The test expects TEST_ENDPOINT to be defined in the Configuration
+ * describing the endpoint of the bucket to which TEST_FS_S3A_NAME points
+ * (f.i. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland).
+ * Evidently, the bucket has to be hosted in the region denoted by the
+ * endpoint for the test to succeed.
+ * <p/>
+ * More info and the list of endpoint identifiers:
+ * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testEndpoint() throws Exception {
+ conf = new Configuration();
+ String endpoint = conf.getTrimmed(TEST_ENDPOINT, "");
+ if (endpoint.isEmpty()) {
+ LOG.warn("Custom endpoint test skipped as " + TEST_ENDPOINT + "config " +
+ "setting was not detected");
+ } else {
+ conf.set(Constants.ENDPOINT, endpoint);
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ AmazonS3Client s3 = fs.getAmazonS3Client();
+ String endPointRegion = "";
+ // Differentiate handling of "s3-" and "s3." based endpoint identifiers
+ String[] endpointParts = StringUtils.split(endpoint, '.');
+ if (endpointParts.length == 3) {
+ endPointRegion = endpointParts[0].substring(3);
+ } else if (endpointParts.length == 4) {
+ endPointRegion = endpointParts[1];
+ } else {
+ fail("Unexpected endpoint");
+ }
+ assertEquals("Endpoint config setting and bucket location differ: ",
+ endPointRegion, s3.getBucketLocation(fs.getUri().getHost()));
+ }
+ }
+
+ @Test
+ public void testProxyConnection() throws Exception {
+ conf = new Configuration();
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.set(Constants.PROXY_HOST, "127.0.0.1");
+ conf.setInt(Constants.PROXY_PORT, 1);
+ String proxy =
+ conf.get(Constants.PROXY_HOST) + ":" + conf.get(Constants.PROXY_PORT);
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server at " + proxy);
+ } catch (AWSClientIOException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testProxyPortWithoutHost() throws Exception {
+ conf = new Configuration();
+ conf.unset(Constants.PROXY_HOST);
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.setInt(Constants.PROXY_PORT, 1);
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a proxy configuration error");
+ } catch (IllegalArgumentException e) {
+ String msg = e.toString();
+ if (!msg.contains(Constants.PROXY_HOST) &&
+ !msg.contains(Constants.PROXY_PORT)) {
+ throw e;
+ }
+ }
+ }
+
+ @Test
+ public void testAutomaticProxyPortSelection() throws Exception {
+ conf = new Configuration();
+ conf.unset(Constants.PROXY_PORT);
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.set(Constants.PROXY_HOST, "127.0.0.1");
+ conf.set(Constants.SECURE_CONNECTIONS, "true");
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server");
+ } catch (AWSClientIOException e) {
+ // expected
+ }
+ conf.set(Constants.SECURE_CONNECTIONS, "false");
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server");
+ } catch (AWSClientIOException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testUsernameInconsistentWithPassword() throws Exception {
+ conf = new Configuration();
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.set(Constants.PROXY_HOST, "127.0.0.1");
+ conf.setInt(Constants.PROXY_PORT, 1);
+ conf.set(Constants.PROXY_USERNAME, "user");
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server");
+ } catch (IllegalArgumentException e) {
+ String msg = e.toString();
+ if (!msg.contains(Constants.PROXY_USERNAME) &&
+ !msg.contains(Constants.PROXY_PASSWORD)) {
+ throw e;
+ }
+ }
+ conf = new Configuration();
+ conf.setInt(Constants.MAX_ERROR_RETRIES, 2);
+ conf.set(Constants.PROXY_HOST, "127.0.0.1");
+ conf.setInt(Constants.PROXY_PORT, 1);
+ conf.set(Constants.PROXY_PASSWORD, "password");
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ fail("Expected a connection error for proxy server");
+ } catch (IllegalArgumentException e) {
+ String msg = e.toString();
+ if (!msg.contains(Constants.PROXY_USERNAME) &&
+ !msg.contains(Constants.PROXY_PASSWORD)) {
+ throw e;
+ }
+ }
+ }
+
+ @Test
+ public void testCredsFromCredentialProvider() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ provisionAccessKeys(conf);
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
+ assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
+ }
+
+ void provisionAccessKeys(final Configuration conf) throws Exception {
+ // add our creds to the provider
+ final CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(Constants.ACCESS_KEY,
+ EXAMPLE_ID.toCharArray());
+ provider.createCredentialEntry(Constants.SECRET_KEY,
+ EXAMPLE_KEY.toCharArray());
+ provider.flush();
+ }
+
+ @Test
+ public void testCredsFromUserInfo() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ provisionAccessKeys(conf);
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ URI uriWithUserInfo = new URI("s3a://123:456@foobar");
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
+ assertEquals("AccessKey incorrect.", "123", creds.getUser());
+ assertEquals("SecretKey incorrect.", "456", creds.getPassword());
+ }
+
+ @Test
+ public void testIDFromUserInfoSecretFromCredentialProvider()
+ throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ provisionAccessKeys(conf);
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ URI uriWithUserInfo = new URI("s3a://123@foobar");
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
+ assertEquals("AccessKey incorrect.", "123", creds.getUser());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
+ }
+
+ @Test
+ public void testSecretFromCredentialProviderIDFromConfig() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ // add our creds to the provider
+ final CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(Constants.SECRET_KEY,
+ EXAMPLE_KEY.toCharArray());
+ provider.flush();
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID);
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
+ assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
+ }
+
+ @Test
+ public void testIDFromCredentialProviderSecretFromConfig() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ jks.toString());
+
+ // add our creds to the provider
+ final CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(Constants.ACCESS_KEY,
+ EXAMPLE_ID.toCharArray());
+ provider.flush();
+
+ conf.set(Constants.SECRET_KEY, EXAMPLE_KEY);
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
+ assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
+ assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
+ }
+
+ @Test
+ public void testExcludingS3ACredentialProvider() throws Exception {
+ // set up conf to have a cred provider
+ final Configuration conf = new Configuration();
+ final File file = tempDir.newFile("test.jks");
+ final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
+ file.toURI());
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ "jceks://s3a/foobar," + jks.toString());
+
+ // first make sure that the s3a based provider is removed
+ Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, S3AFileSystem.class);
+ String newPath = conf.get(
+ CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
+ assertFalse("Provider Path incorrect", newPath.contains("s3a://"));
+
+ // now let's make sure the new path is created by the S3AFileSystem
+ // and the integration still works. Let's provision the keys through
+ // the altered configuration instance and then try and access them
+ // using the original config with the s3a provider in the path.
+ provisionAccessKeys(c);
+
+ conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
+ URI uriWithUserInfo = new URI("s3a://123:456@foobar");
+ S3xLoginHelper.Login creds =
+ S3AUtils.getAWSAccessKeys(uriWithUserInfo, conf);
+ assertEquals("AccessKey incorrect.", "123", creds.getUser());
+ assertEquals("SecretKey incorrect.", "456", creds.getPassword());
+
+ }
+
+ @Test
+ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty()
+ throws Exception {
+
+ conf = new Configuration();
+ conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true));
+ assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false));
+
+ try {
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ assertNotNull(fs);
+ AmazonS3Client s3 = fs.getAmazonS3Client();
+ assertNotNull(s3);
+ S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class,
+ "clientOptions");
+ assertTrue("Expected to find path style access to be switched on!",
+ clientOptions.isPathStyleAccess());
+ byte[] file = ContractTestUtils.toAsciiByteArray("test file");
+ ContractTestUtils.writeAndRead(fs,
+ new Path("/path/style/access/testFile"), file, file.length,
+ conf.getInt(Constants.FS_S3A_BLOCK_SIZE, file.length), false, true);
+ } catch (final AWSS3IOException e) {
+ LOG.error("Caught exception: ", e);
+ // Catch/pass standard path style access behaviour when live bucket
+ // isn't in the same region as the s3 client default. See
+ // http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
+ assertEquals(e.getStatusCode(), HttpStatus.SC_MOVED_PERMANENTLY);
+ }
+ }
+
+ @Test
+ public void testDefaultUserAgent() throws Exception {
+ conf = new Configuration();
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ assertNotNull(fs);
+ AmazonS3Client s3 = fs.getAmazonS3Client();
+ assertNotNull(s3);
+ ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
+ "clientConfiguration");
+ assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent());
+ }
+
+ @Test
+ public void testCustomUserAgent() throws Exception {
+ conf = new Configuration();
+ conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ assertNotNull(fs);
+ AmazonS3Client s3 = fs.getAmazonS3Client();
+ assertNotNull(s3);
+ ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
+ "clientConfiguration");
+ assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(),
+ awsConf.getUserAgent());
+ }
+
+ /**
+ * Reads and returns a field from an object using reflection. If the field
+ * cannot be found, is null, or is not the expected type, then this method
+ * fails the test.
+ *
+ * @param target object to read
+ * @param fieldType type of field to read, which will also be the return type
+ * @param fieldName name of field to read
+ * @return field that was read
+ * @throws IllegalAccessException if access not allowed
+ */
+ private static <T> T getField(Object target, Class<T> fieldType,
+ String fieldName) throws IllegalAccessException {
+ Object obj = FieldUtils.readField(target, fieldName, true);
+ assertNotNull(String.format(
+ "Could not read field named %s in object with class %s.", fieldName,
+ target.getClass().getName()), obj);
+ assertTrue(String.format(
+ "Unexpected type found for field named %s, expected %s, actual %s.",
+ fieldName, fieldType.getName(), obj.getClass().getName()),
+ fieldType.isAssignableFrom(obj.getClass()));
+ return fieldType.cast(obj);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java
new file mode 100644
index 0000000..b3d7abf
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACredentialsInURL.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URLEncoder;
+import java.nio.file.AccessDeniedException;
+
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.TEST_FS_S3A_NAME;
+
+/**
+ * Tests that credentials can go into the URL. This includes a valid
+ * set, and a check that an invalid set do at least get stripped out
+ * of the final URI
+ */
+public class ITestS3ACredentialsInURL extends Assert {
+ private S3AFileSystem fs;
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3ACredentialsInURL.class);
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @After
+ public void teardown() {
+ IOUtils.closeStream(fs);
+ }
+
+ /**
+ * Test instantiation.
+ * @throws Throwable
+ */
+ @Test
+ public void testInstantiateFromURL() throws Throwable {
+
+ Configuration conf = new Configuration();
+ String accessKey = conf.get(Constants.ACCESS_KEY);
+ String secretKey = conf.get(Constants.SECRET_KEY);
+ String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, "");
+ Assume.assumeNotNull(fsname, accessKey, secretKey);
+ URI original = new URI(fsname);
+ URI secretsURI = createUriWithEmbeddedSecrets(original,
+ accessKey, secretKey);
+ if (secretKey.contains("/")) {
+ assertTrue("test URI encodes the / symbol", secretsURI.toString().
+ contains("%252F"));
+ }
+ if (secretKey.contains("+")) {
+ assertTrue("test URI encodes the + symbol", secretsURI.toString().
+ contains("%252B"));
+ }
+ assertFalse("Does not contain secrets", original.equals(secretsURI));
+
+ conf.set(TEST_FS_S3A_NAME, secretsURI.toString());
+ conf.unset(Constants.ACCESS_KEY);
+ conf.unset(Constants.SECRET_KEY);
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ String fsURI = fs.getUri().toString();
+ assertFalse("FS URI contains a @ symbol", fsURI.contains("@"));
+ assertFalse("FS URI contains a % symbol", fsURI.contains("%"));
+ if (!original.toString().startsWith(fsURI)) {
+ fail("Filesystem URI does not match original");
+ }
+ validate("original path", new Path(original));
+ validate("bare path", new Path("/"));
+ validate("secrets path", new Path(secretsURI));
+ }
+
+ private void validate(String text, Path path) throws IOException {
+ try {
+ fs.canonicalizeUri(path.toUri());
+ fs.checkPath(path);
+ assertTrue(text + " Not a directory",
+ fs.getFileStatus(new Path("/")).isDirectory());
+ fs.globStatus(path);
+ } catch (AssertionError e) {
+ throw e;
+ } catch (Exception e) {
+ LOG.debug("{} failure: {}", text, e, e);
+ fail(text + " Test failed");
+ }
+ }
+
+ /**
+ * Set up some invalid credentials, verify login is rejected.
+ * @throws Throwable
+ */
+ @Test
+ public void testInvalidCredentialsFail() throws Throwable {
+ Configuration conf = new Configuration();
+ String fsname = conf.getTrimmed(TEST_FS_S3A_NAME, "");
+ Assume.assumeNotNull(fsname);
+ URI original = new URI(fsname);
+ URI testURI = createUriWithEmbeddedSecrets(original, "user", "//");
+
+ conf.set(TEST_FS_S3A_NAME, testURI.toString());
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ try {
+ S3AFileStatus status = fs.getFileStatus(new Path("/"));
+ fail("Expected an AccessDeniedException, got " + status);
+ } catch (AccessDeniedException e) {
+ // expected
+ }
+
+ }
+
+ private URI createUriWithEmbeddedSecrets(URI original,
+ String accessKey,
+ String secretKey) throws UnsupportedEncodingException {
+ String encodedSecretKey = URLEncoder.encode(secretKey, "UTF-8");
+ String formattedString = String.format("%s://%s:%s@%s/%s/",
+ original.getScheme(),
+ accessKey,
+ encodedSecretKey,
+ original.getHost(),
+ original.getPath());
+ URI testURI;
+ try {
+ testURI = new Path(formattedString).toUri();
+ } catch (IllegalArgumentException e) {
+ // inner cause is stripped to keep any secrets out of stack traces
+ throw new IllegalArgumentException("Could not encode Path");
+ }
+ return testURI;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java
new file mode 100644
index 0000000..4543278
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+
+/**
+ * Test whether or not encryption works by turning it on. Some checks
+ * are made for different file sizes as there have been reports that the
+ * file length may be rounded up to match word boundaries.
+ */
+public class ITestS3AEncryption extends AbstractS3ATestBase {
+ private static final String AES256 = Constants.SERVER_SIDE_ENCRYPTION_AES256;
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ AES256);
+ return conf;
+ }
+
+ private static final int[] SIZES = {
+ 0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 10 - 3, 2 ^ 11 - 2, 2 ^ 12 - 1
+ };
+
+ @Override
+ public void teardown() throws Exception {
+ super.teardown();
+ IOUtils.closeStream(getFileSystem());
+ }
+
+ @Test
+ public void testEncryption() throws Throwable {
+ for (int size: SIZES) {
+ validateEncryptionForFilesize(size);
+ }
+ }
+
+ @Test
+ public void testEncryptionOverRename() throws Throwable {
+ skipIfEncryptionTestsDisabled(getConfiguration());
+ Path src = path(createFilename(1024));
+ byte[] data = dataset(1024, 'a', 'z');
+ S3AFileSystem fs = getFileSystem();
+ writeDataset(fs, src, data, data.length, 1024 * 1024, true);
+ ContractTestUtils.verifyFileContents(fs, src, data);
+ Path dest = path(src.getName() + "-copy");
+ fs.rename(src, dest);
+ ContractTestUtils.verifyFileContents(fs, dest, data);
+ assertEncrypted(dest);
+ }
+
+ protected void validateEncryptionForFilesize(int len) throws IOException {
+ skipIfEncryptionTestsDisabled(getConfiguration());
+ describe("Create an encrypted file of size " + len);
+ String src = createFilename(len);
+ Path path = writeThenReadFile(src, len);
+ assertEncrypted(path);
+ rm(getFileSystem(), path, false, false);
+ }
+
+ private String createFilename(int len) {
+ return String.format("%s-%04x", methodName.getMethodName(), len);
+ }
+
+ /**
+ * Assert that at path references an encrypted blob.
+ * @param path path
+ * @throws IOException on a failure
+ */
+ private void assertEncrypted(Path path) throws IOException {
+ ObjectMetadata md = getFileSystem().getObjectMetadata(path);
+ assertEquals(AES256, md.getSSEAlgorithm());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java
new file mode 100644
index 0000000..81578c2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+
+/**
+ * Test whether or not encryption settings propagate by choosing an invalid
+ * one. We expect the write to fail with a 400 bad request error
+ */
+public class ITestS3AEncryptionAlgorithmPropagation
+ extends AbstractS3ATestBase {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ "DES");
+ return conf;
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ super.teardown();
+ IOUtils.closeStream(getFileSystem());
+ }
+
+ @Test
+ public void testEncrypt0() throws Throwable {
+ writeThenReadFileToFailure(0);
+ }
+
+ @Test
+ public void testEncrypt256() throws Throwable {
+ writeThenReadFileToFailure(256);
+ }
+
+ /**
+ * Make this a no-op so test setup doesn't fail.
+ * @param path path path
+ * @throws IOException on any failure
+ */
+ @Override
+ protected void mkdirs(Path path) throws IOException {
+
+ }
+
+ protected void writeThenReadFileToFailure(int len) throws IOException {
+ skipIfEncryptionTestsDisabled(getConfiguration());
+ describe("Create an encrypted file of size " + len);
+ try {
+ writeThenReadFile(methodName.getMethodName() + '-' + len, len);
+ fail("Expected an exception about an illegal encryption algorithm");
+ } catch (AWSS3IOException e) {
+ assertStatusCode(e, 400);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionFastOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionFastOutputStream.java
new file mode 100644
index 0000000..c06fed1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionFastOutputStream.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Run the encryption tests against the Fast output stream.
+ * This verifies that both file writing paths can encrypt their data.
+ */
+public class ITestS3AEncryptionFastOutputStream extends ITestS3AEncryption {
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.setBoolean(Constants.FAST_UPLOAD, true);
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
new file mode 100644
index 0000000..d8e017e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.nio.file.AccessDeniedException;
+import java.util.concurrent.Callable;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+
+/**
+ * Test S3A Failure translation, including a functional test
+ * generating errors during stream IO.
+ */
+public class ITestS3AFailureHandling extends AbstractFSContractTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AFailureHandling.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Test
+ public void testReadFileChanged() throws Throwable {
+ describe("overwrite a file with a shorter one during a read, seek");
+ final int fullLength = 8192;
+ final byte[] fullDataset = dataset(fullLength, 'a', 32);
+ final int shortLen = 4096;
+ final byte[] shortDataset = dataset(shortLen, 'A', 32);
+ final FileSystem fs = getFileSystem();
+ final Path testpath = path("readFileToChange.txt");
+ // initial write
+ writeDataset(fs, testpath, fullDataset, fullDataset.length, 1024, false);
+ try(FSDataInputStream instream = fs.open(testpath)) {
+ instream.seek(fullLength - 16);
+ assertTrue("no data to read", instream.read() >= 0);
+ // overwrite
+ writeDataset(fs, testpath, shortDataset, shortDataset.length, 1024, true);
+ // here the file length is less. Probe the file to see if this is true,
+ // with a spin and wait
+ eventually(30 *1000, new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ assertEquals(shortLen, fs.getFileStatus(testpath).getLen());
+ return null;
+ }
+ });
+ // here length is shorter. Assuming it has propagated to all replicas,
+ // the position of the input stream is now beyond the EOF.
+ // An attempt to seek backwards to a position greater than the
+ // short length will raise an exception from AWS S3, which must be
+ // translated into an EOF
+
+ instream.seek(shortLen + 1024);
+ int c = instream.read();
+ assertIsEOF("read()", c);
+
+ byte[] buf = new byte[256];
+
+ assertIsEOF("read(buffer)", instream.read(buf));
+ assertIsEOF("read(offset)",
+ instream.read(instream.getPos(), buf, 0, buf.length));
+
+ // now do a block read fully, again, backwards from the current pos
+ try {
+ instream.readFully(shortLen + 512, buf);
+ fail("Expected readFully to fail");
+ } catch (EOFException expected) {
+ LOG.debug("Expected EOF: ", expected);
+ }
+
+ assertIsEOF("read(offset)",
+ instream.read(shortLen + 510, buf, 0, buf.length));
+
+ // seek somewhere useful
+ instream.seek(shortLen - 256);
+
+ // delete the file. Reads must fail
+ fs.delete(testpath, false);
+
+ try {
+ int r = instream.read();
+ fail("Expected an exception, got " + r);
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+
+ try {
+ instream.readFully(2048, buf);
+ fail("Expected readFully to fail");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+
+ }
+ }
+
+ /**
+ * Assert that a read operation returned an EOF value.
+ * @param operation specific operation
+ * @param readResult result
+ */
+ private void assertIsEOF(String operation, int readResult) {
+ assertEquals("Expected EOF from "+ operation
+ + "; got char " + (char) readResult, -1, readResult);
+ }
+
+ @Test
+ public void test404isNotFound() throws Throwable {
+ verifyTranslated(FileNotFoundException.class, createS3Exception(404));
+ }
+
+ protected Exception verifyTranslated(Class clazz,
+ AmazonClientException exception) throws Exception {
+ return verifyExceptionClass(clazz,
+ translateException("test", "/", exception));
+ }
+
+ @Test
+ public void test401isNotPermittedFound() throws Throwable {
+ verifyTranslated(AccessDeniedException.class,
+ createS3Exception(401));
+ }
+
+ protected AmazonS3Exception createS3Exception(int code) {
+ AmazonS3Exception source = new AmazonS3Exception("");
+ source.setStatusCode(code);
+ return source;
+ }
+
+ @Test
+ public void testGenericS3Exception() throws Throwable {
+ // S3 exception of no known type
+ AWSS3IOException ex = (AWSS3IOException)verifyTranslated(
+ AWSS3IOException.class,
+ createS3Exception(451));
+ assertEquals(451, ex.getStatusCode());
+ }
+
+ @Test
+ public void testGenericServiceS3Exception() throws Throwable {
+ // service exception of no known type
+ AmazonServiceException ase = new AmazonServiceException("unwind");
+ ase.setStatusCode(500);
+ AWSServiceIOException ex = (AWSServiceIOException)verifyTranslated(
+ AWSServiceIOException.class,
+ ase);
+ assertEquals(500, ex.getStatusCode());
+ }
+
+ @Test
+ public void testGenericClientException() throws Throwable {
+ // Generic Amazon exception
+ verifyTranslated(AWSClientIOException.class,
+ new AmazonClientException(""));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFastOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFastOutputStream.java
new file mode 100644
index 0000000..b5fa1c3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFastOutputStream.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+/**
+ * Tests regular and multi-part upload functionality for S3AFastOutputStream.
+ * File sizes are kept small to reduce test duration on slow connections
+ */
+public class ITestS3AFastOutputStream {
+ private FileSystem fs;
+
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024);
+ conf.setInt(Constants.MULTIPART_SIZE, 5 * 1024 * 1024);
+ conf.setBoolean(Constants.FAST_UPLOAD, true);
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(getTestPath(), true);
+ }
+ }
+
+ protected Path getTestPath() {
+ return new Path("/tests3a");
+ }
+
+ @Test
+ public void testRegularUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
+ }
+
+ @Test
+ public void testMultiPartUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 *
+ 1024);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
new file mode 100644
index 0000000..2a6ba0c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.net.URI;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Statistic.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.MetricDiff;
+import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
+
+/**
+ * Use metrics to assert about the cost of file status queries.
+ * {@link S3AFileSystem#getFileStatus(Path)}.
+ */
+public class ITestS3AFileOperationCost extends AbstractFSContractTestBase {
+
+ private MetricDiff metadataRequests;
+ private MetricDiff listRequests;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AFileOperationCost.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ @Override
+ public S3AFileSystem getFileSystem() {
+ return (S3AFileSystem) super.getFileSystem();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ S3AFileSystem fs = getFileSystem();
+ metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
+ listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnFile() throws Throwable {
+ describe("performing getFileStatus on a file");
+ Path simpleFile = path("simple.txt");
+ S3AFileSystem fs = getFileSystem();
+ touch(fs, simpleFile);
+ resetMetricDiffs();
+ S3AFileStatus status = fs.getFileStatus(simpleFile);
+ assertTrue("not a file: " + status, status.isFile());
+ metadataRequests.assertDiffEquals(1);
+ listRequests.assertDiffEquals(0);
+ }
+
+ private void resetMetricDiffs() {
+ reset(metadataRequests, listRequests);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnEmptyDir() throws Throwable {
+ describe("performing getFileStatus on an empty directory");
+ S3AFileSystem fs = getFileSystem();
+ Path dir = path("empty");
+ fs.mkdirs(dir);
+ resetMetricDiffs();
+ S3AFileStatus status = fs.getFileStatus(dir);
+ assertTrue("not empty: " + status, status.isEmptyDirectory());
+ metadataRequests.assertDiffEquals(2);
+ listRequests.assertDiffEquals(0);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnMissingFile() throws Throwable {
+ describe("performing getFileStatus on a missing file");
+ S3AFileSystem fs = getFileSystem();
+ Path path = path("missing");
+ resetMetricDiffs();
+ try {
+ S3AFileStatus status = fs.getFileStatus(path);
+ fail("Got a status back from a missing file path " + status);
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+ metadataRequests.assertDiffEquals(2);
+ listRequests.assertDiffEquals(1);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnMissingSubPath() throws Throwable {
+ describe("performing getFileStatus on a missing file");
+ S3AFileSystem fs = getFileSystem();
+ Path path = path("missingdir/missingpath");
+ resetMetricDiffs();
+ try {
+ S3AFileStatus status = fs.getFileStatus(path);
+ fail("Got a status back from a missing file path " + status);
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+ metadataRequests.assertDiffEquals(2);
+ listRequests.assertDiffEquals(1);
+ }
+
+ @Test
+ public void testCostOfGetFileStatusOnNonEmptyDir() throws Throwable {
+ describe("performing getFileStatus on a non-empty directory");
+ S3AFileSystem fs = getFileSystem();
+ Path dir = path("empty");
+ fs.mkdirs(dir);
+ Path simpleFile = new Path(dir, "simple.txt");
+ touch(fs, simpleFile);
+ resetMetricDiffs();
+ S3AFileStatus status = fs.getFileStatus(dir);
+ if (status.isEmptyDirectory()) {
+ // erroneous state
+ String fsState = fs.toString();
+ fail("FileStatus says directory isempty: " + status
+ + "\n" + ContractTestUtils.ls(fs, dir)
+ + "\n" + fsState);
+ }
+ metadataRequests.assertDiffEquals(2);
+ listRequests.assertDiffEquals(1);
+ }
+
+ @Test
+ public void testCostOfCopyFromLocalFile() throws Throwable {
+ describe("testCostOfCopyFromLocalFile");
+ File localTestDir = getTestDir("tmp");
+ localTestDir.mkdirs();
+ File tmpFile = File.createTempFile("tests3acost", ".txt",
+ localTestDir);
+ tmpFile.delete();
+ try {
+ URI localFileURI = tmpFile.toURI();
+ FileSystem localFS = FileSystem.get(localFileURI,
+ getFileSystem().getConf());
+ Path localPath = new Path(localFileURI);
+ int len = 10 * 1024;
+ byte[] data = dataset(len, 'A', 'Z');
+ writeDataset(localFS, localPath, data, len, 1024, true);
+ S3AFileSystem s3a = getFileSystem();
+ MetricDiff copyLocalOps = new MetricDiff(s3a,
+ INVOCATION_COPY_FROM_LOCAL_FILE);
+ MetricDiff putRequests = new MetricDiff(s3a,
+ OBJECT_PUT_REQUESTS);
+ MetricDiff putBytes = new MetricDiff(s3a,
+ OBJECT_PUT_BYTES);
+
+ Path remotePath = path("copied");
+ s3a.copyFromLocalFile(false, true, localPath, remotePath);
+ verifyFileContents(s3a, remotePath, data);
+ copyLocalOps.assertDiffEquals(1);
+ putRequests.assertDiffEquals(1);
+ putBytes.assertDiffEquals(len);
+ // print final stats
+ LOG.info("Filesystem {}", s3a);
+ } finally {
+ tmpFile.delete();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
new file mode 100644
index 0000000..858ac22
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Tests a live S3 system. If your keys and bucket aren't specified, all tests
+ * are marked as passed.
+ *
+ * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
+ * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
+ * properly making it impossible to skip the tests if we don't have a valid
+ * bucket.
+ **/
+public class ITestS3AFileSystemContract extends FileSystemContractBaseTest {
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3AFileSystemContract.class);
+
+ @Override
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+
+ fs = S3ATestUtils.createTestFileSystem(conf);
+ super.setUp();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(path("test"), true);
+ }
+ super.tearDown();
+ }
+
+ @Override
+ public void testMkdirsWithUmask() throws Exception {
+ // not supported
+ }
+
+ @Override
+ public void testRenameFileAsExistingFile() throws Exception {
+ if (!renameSupported()) {
+ return;
+ }
+
+ Path src = path("/test/hadoop/file");
+ createFile(src);
+ Path dst = path("/test/new/newfile");
+ createFile(dst);
+ // s3 doesn't support rename option
+ // rename-overwrites-dest is always allowed.
+ rename(src, dst, true, false, true);
+ }
+
+ @Override
+ public void testRenameDirectoryAsExistingDirectory() throws Exception {
+ if (!renameSupported()) {
+ return;
+ }
+
+ Path src = path("/test/hadoop/dir");
+ fs.mkdirs(src);
+ createFile(path("/test/hadoop/dir/file1"));
+ createFile(path("/test/hadoop/dir/subdir/file2"));
+
+ Path dst = path("/test/new/newdir");
+ fs.mkdirs(dst);
+ rename(src, dst, true, false, true);
+ assertFalse("Nested file1 exists",
+ fs.exists(path("/test/hadoop/dir/file1")));
+ assertFalse("Nested file2 exists",
+ fs.exists(path("/test/hadoop/dir/subdir/file2")));
+ assertTrue("Renamed nested file1 exists",
+ fs.exists(path("/test/new/newdir/file1")));
+ assertTrue("Renamed nested exists",
+ fs.exists(path("/test/new/newdir/subdir/file2")));
+ }
+
+// @Override
+ public void testMoveDirUnderParent() throws Throwable {
+ // not support because
+ // Fails if dst is a directory that is not empty.
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
new file mode 100644
index 0000000..360a151
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.net.URI;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
+import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest;
+import com.amazonaws.services.securitytoken.model.GetSessionTokenResult;
+import com.amazonaws.services.securitytoken.model.Credentials;
+
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.fs.s3native.S3xLoginHelper;
+import org.apache.hadoop.conf.Configuration;
+
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * Tests use of temporary credentials (for example, AWS STS & S3).
+ * This test extends a class that "does things to the root directory", and
+ * should only be used against transient filesystems where you don't care about
+ * the data.
+ */
+public class ITestS3ATemporaryCredentials extends AbstractFSContractTestBase {
+ public static final String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled";
+ public static final String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestS3ATemporaryCredentials.class);
+
+ private static final String PROVIDER_CLASS
+ = TemporaryAWSCredentialsProvider.NAME;
+
+ private static final long TEST_FILE_SIZE = 1024;
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ /**
+ * Test use of STS for requesting temporary credentials.
+ *
+ * The property test.sts.endpoint can be set to point this at different
+ * STS endpoints. This test will use the AWS credentials (if provided) for
+ * S3A tests to request temporary credentials, then attempt to use those
+ * credentials instead.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testSTS() throws IOException {
+ Configuration conf = getContract().getConf();
+ if (!conf.getBoolean(TEST_STS_ENABLED, true)) {
+ skip("STS functional tests disabled");
+ }
+
+ S3xLoginHelper.Login login = S3AUtils.getAWSAccessKeys(
+ URI.create("s3a://foobar"), conf);
+ if (!login.hasLogin()) {
+ skip("testSTS disabled because AWS credentials not configured");
+ }
+ AWSCredentialsProvider parentCredentials = new BasicAWSCredentialsProvider(
+ login.getUser(), login.getPassword());
+
+ String stsEndpoint = conf.getTrimmed(TEST_STS_ENDPOINT, "");
+ AWSSecurityTokenServiceClient stsClient;
+ stsClient = new AWSSecurityTokenServiceClient(parentCredentials);
+ if (!stsEndpoint.isEmpty()) {
+ LOG.debug("STS Endpoint ={}", stsEndpoint);
+ stsClient.setEndpoint(stsEndpoint);
+ }
+ GetSessionTokenRequest sessionTokenRequest = new GetSessionTokenRequest();
+ sessionTokenRequest.setDurationSeconds(900);
+ GetSessionTokenResult sessionTokenResult;
+ sessionTokenResult = stsClient.getSessionToken(sessionTokenRequest);
+ Credentials sessionCreds = sessionTokenResult.getCredentials();
+
+ String childAccessKey = sessionCreds.getAccessKeyId();
+ conf.set(ACCESS_KEY, childAccessKey);
+ String childSecretKey = sessionCreds.getSecretAccessKey();
+ conf.set(SECRET_KEY, childSecretKey);
+ String sessionToken = sessionCreds.getSessionToken();
+ conf.set(SESSION_TOKEN, sessionToken);
+
+ conf.set(AWS_CREDENTIALS_PROVIDER, PROVIDER_CLASS);
+
+ try(S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
+ createAndVerifyFile(fs, path("testSTS"), TEST_FILE_SIZE);
+ }
+
+ // now create an invalid set of credentials by changing the session
+ // token
+ conf.set(SESSION_TOKEN, "invalid-" + sessionToken);
+ try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
+ createAndVerifyFile(fs, path("testSTSInvalidToken"), TEST_FILE_SIZE);
+ fail("Expected an access exception, but file access to "
+ + fs.getUri() + " was allowed: " + fs);
+ } catch (AWSS3IOException ex) {
+ LOG.info("Expected Exception: {}", ex.toString());
+ LOG.debug("Expected Exception: {}", ex, ex);
+ }
+ }
+
+ @Test
+ public void testTemporaryCredentialValidation() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(ACCESS_KEY, "accesskey");
+ conf.set(SECRET_KEY, "secretkey");
+ conf.set(SESSION_TOKEN, "");
+ TemporaryAWSCredentialsProvider provider
+ = new TemporaryAWSCredentialsProvider(getFileSystem().getUri(), conf);
+ try {
+ AWSCredentials credentials = provider.getCredentials();
+ fail("Expected a CredentialInitializationException,"
+ + " got " + credentials);
+ } catch (CredentialInitializationException expected) {
+ // expected
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9c346e/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBlockingThreadPoolExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBlockingThreadPoolExecutorService.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBlockingThreadPoolExecutorService.java
deleted file mode 100644
index 25a8958..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestBlockingThreadPoolExecutorService.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import org.apache.hadoop.util.StopWatch;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-/**
- * Basic unit test for S3A's blocking executor service.
- */
-public class TestBlockingThreadPoolExecutorService {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- BlockingThreadPoolExecutorService.class);
-
- private static final int NUM_ACTIVE_TASKS = 4;
- private static final int NUM_WAITING_TASKS = 2;
- private static final int TASK_SLEEP_MSEC = 100;
- private static final int SHUTDOWN_WAIT_MSEC = 200;
- private static final int SHUTDOWN_WAIT_TRIES = 5;
- private static final int BLOCKING_THRESHOLD_MSEC = 50;
-
- private static final Integer SOME_VALUE = 1337;
-
- private static BlockingThreadPoolExecutorService tpe = null;
-
- @AfterClass
- public static void afterClass() throws Exception {
- ensureDestroyed();
- }
-
- /**
- * Basic test of running one trivial task.
- */
- @Test
- public void testSubmitCallable() throws Exception {
- ensureCreated();
- ListenableFuture<Integer> f = tpe.submit(callableSleeper);
- Integer v = f.get();
- assertEquals(SOME_VALUE, v);
- }
-
- /**
- * More involved test, including detecting blocking when at capacity.
- */
- @Test
- public void testSubmitRunnable() throws Exception {
- ensureCreated();
- int totalTasks = NUM_ACTIVE_TASKS + NUM_WAITING_TASKS;
- StopWatch stopWatch = new StopWatch().start();
- for (int i = 0; i < totalTasks; i++) {
- tpe.submit(sleeper);
- assertDidntBlock(stopWatch);
- }
- tpe.submit(sleeper);
- assertDidBlock(stopWatch);
- }
-
- @Test
- public void testShutdown() throws Exception {
- // Cover create / destroy, regardless of when this test case runs
- ensureCreated();
- ensureDestroyed();
-
- // Cover create, execute, destroy, regardless of when test case runs
- ensureCreated();
- testSubmitRunnable();
- ensureDestroyed();
- }
-
- // Helper functions, etc.
-
- private void assertDidntBlock(StopWatch sw) {
- try {
- assertFalse("Non-blocking call took too long.",
- sw.now(TimeUnit.MILLISECONDS) > BLOCKING_THRESHOLD_MSEC);
- } finally {
- sw.reset().start();
- }
- }
-
- private void assertDidBlock(StopWatch sw) {
- try {
- if (sw.now(TimeUnit.MILLISECONDS) < BLOCKING_THRESHOLD_MSEC) {
- throw new RuntimeException("Blocking call returned too fast.");
- }
- } finally {
- sw.reset().start();
- }
- }
-
- private Runnable sleeper = new Runnable() {
- @Override
- public void run() {
- String name = Thread.currentThread().getName();
- try {
- Thread.sleep(TASK_SLEEP_MSEC);
- } catch (InterruptedException e) {
- LOG.info("Thread {} interrupted.", name);
- Thread.currentThread().interrupt();
- }
- }
- };
-
- private Callable<Integer> callableSleeper = new Callable<Integer>() {
- @Override
- public Integer call() throws Exception {
- sleeper.run();
- return SOME_VALUE;
- }
- };
-
- /**
- * Helper function to create thread pool under test.
- */
- private static void ensureCreated() throws Exception {
- if (tpe == null) {
- LOG.debug("Creating thread pool");
- tpe = new BlockingThreadPoolExecutorService(NUM_ACTIVE_TASKS,
- NUM_WAITING_TASKS, 1, TimeUnit.SECONDS, "btpetest");
- }
- }
-
- /**
- * Helper function to terminate thread pool under test, asserting that
- * shutdown -> terminate works as expected.
- */
- private static void ensureDestroyed() throws Exception {
- if (tpe == null) {
- return;
- }
- int shutdownTries = SHUTDOWN_WAIT_TRIES;
-
- tpe.shutdown();
- if (!tpe.isShutdown()) {
- throw new RuntimeException("Shutdown had no effect.");
- }
-
- while (!tpe.awaitTermination(SHUTDOWN_WAIT_MSEC,
- TimeUnit.MILLISECONDS)) {
- LOG.info("Waiting for thread pool shutdown.");
- if (shutdownTries-- <= 0) {
- LOG.error("Failed to terminate thread pool gracefully.");
- break;
- }
- }
- if (!tpe.isTerminated()) {
- tpe.shutdownNow();
- if (!tpe.awaitTermination(SHUTDOWN_WAIT_MSEC,
- TimeUnit.MILLISECONDS)) {
- throw new RuntimeException(
- "Failed to terminate thread pool in timely manner.");
- }
- }
- tpe = null;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/52] [abbrv] hadoop git commit: HADOOP-13428. Fixed hadoop-common
build files to generate jdiff by working around a jdiff-bug. Contributed by
Wangda Tan.
Posted by aw...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99603e90/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.7.2.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.7.2.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.7.2.xml
new file mode 100644
index 0000000..5ef99b2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.7.2.xml
@@ -0,0 +1,46648 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Thu Aug 18 16:00:16 PDT 2016 -->
+
+<api
+ xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+ xsi:noNamespaceSchemaLocation='api.xsd'
+ name="Apache Hadoop Common 2.7.2"
+ jdversion="1.0.9">
+
+<!-- Command line arguments = -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/hadoop-annotations.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/jdiff.jar -verbose -classpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/classes:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.7.2.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/lib/tools.jar:/Users/wtan/.m2/repository/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/Users/wtan/.m2/repository/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/Users/wtan/.m2/repository/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/Users/wtan/.m2/repository/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/Users/wtan/.m2/repository/commons-
httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/Users/wtan/.m2/repository/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/Users/wtan/.m2/repository/commons-io/commons-io/2.4/commons-io-2.4.jar:/Users/wtan/.m2/repository/commons-net/commons-net/3.1/commons-net-3.1.jar:/Users/wtan/.m2/repository/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/Users/wtan/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/Users/wtan/.m2/repository/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/Users/wtan/.m2/repository/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/Users/wtan/.m2/repository/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar:/Users/wtan/.m2/repository/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/Users/wtan/.m2/repository/javax/xml/bind/jaxb-ap
i/2.2.2/jaxb-api-2.2.2.jar:/Users/wtan/.m2/repository/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/Users/wtan/.m2/repository/javax/activation/activation/1.1/activation-1.1.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/Users/wtan/.m2/repository/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/Users/wtan/.m2/repository/asm/asm/3.2/asm-3.2.jar:/Users/wtan/.m2/repository/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/Users/wtan/.m2/repository/log4j/log4j/1.2.17/log4j-1.2.17.jar:/Users/wtan/.m2/repository/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/Users/wtan/.m2/repository/org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar:/Users/wtan/.m2/repository/org/apache/httpcomponents/httpcore/4.2.5/httpcore-4.2.5.jar:/Users/wtan/.m2/repository/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.ja
r:/Users/wtan/.m2/repository/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/Users/wtan/.m2/repository/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/Users/wtan/.m2/repository/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/Users/wtan/.m2/repository/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/Users/wtan/.m2/repository/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/Users/wtan/.m2/repository/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/Users/wtan/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/Users/wtan/.m2/repository/org/apache/avro/avro/1.7.4/avro-1.7.4.jar:/Users/wtan/.m2/repository/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/Users/wtan/.m2/repository/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1
.jar:/Users/wtan/.m2/repository/org/apache/ant/ant/1.8.1/ant-1.8.1.jar:/Users/wtan/.m2/repository/org/apache/ant/ant-launcher/1.8.1/ant-launcher-1.8.1.jar:/Users/wtan/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/Users/wtan/.m2/repository/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.7.2.jar:/Users/wtan/.m2/repository/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/wtan/.m2/repository/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/Users/wtan/.m2/repository/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/Users/wtan/.m2/repository/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar:/Users/wtan/.m2/repository/com/jcraft/j
sch/0.1.42/jsch-0.1.42.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar:/Users/wtan/.m2/repository/org/apache/curator/curator-recipes/2.7.1/curator-recipes-2.7.1.jar:/Users/wtan/.m2/repository/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar:/Users/wtan/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar:/Users/wtan/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/Users/wtan/.m2/repository/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/Users/wtan/.m2/repository/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/Users/wtan/.m2/repository/org/tukaani/xz/1.0/xz-1.0.jar -sourcepath /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/src/main/java -doclet org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsJDiffDoclet -docletpath /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-c
ommon/target/hadoop-annotations.jar:/Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/jdiff.jar -apidir /Users/wtan/project/github/hadoop-common-trunk/hadoop-common-project/hadoop-common/target/site/jdiff/xml -apiname Apache Hadoop Common 2.7.2 -->
+<package name="org.apache.hadoop">
+ <!-- start class org.apache.hadoop.HadoopIllegalArgumentException -->
+ <class name="HadoopIllegalArgumentException" extends="java.lang.IllegalArgumentException"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="HadoopIllegalArgumentException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Constructs exception with the specified detail message.
+ @param message detailed message.]]>
+ </doc>
+ </constructor>
+ <doc>
+ <![CDATA[Indicates that a method has been passed illegal or invalid argument. This
+ exception is thrown instead of IllegalArgumentException to differentiate the
+ exception thrown in Hadoop implementation from the one thrown in JDK.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.HadoopIllegalArgumentException -->
+</package>
+<package name="org.apache.hadoop.conf">
+ <!-- start interface org.apache.hadoop.conf.Configurable -->
+ <interface name="Configurable" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <method name="setConf"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Set the configuration to be used by this object.]]>
+ </doc>
+ </method>
+ <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return the configuration used by this object.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Something that may be configured with a {@link Configuration}.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.conf.Configurable -->
+ <!-- start class org.apache.hadoop.conf.Configuration -->
+ <class name="Configuration" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="java.lang.Iterable"/>
+ <implements name="org.apache.hadoop.io.Writable"/>
+ <constructor name="Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[A new configuration.]]>
+ </doc>
+ </constructor>
+ <constructor name="Configuration" type="boolean"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[A new configuration where the behavior of reading from the default
+ resources can be turned off.
+
+ If the parameter {@code loadDefaults} is false, the new instance
+ will not load resources from the default files.
+ @param loadDefaults specifies whether to load from the default files]]>
+ </doc>
+ </constructor>
+ <constructor name="Configuration" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[A new configuration with the same settings cloned from another.
+
+ @param other the configuration from which to clone settings.]]>
+ </doc>
+ </constructor>
+ <method name="addDeprecations"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="deltas" type="org.apache.hadoop.conf.Configuration.DeprecationDelta[]"/>
+ <doc>
+ <![CDATA[Adds a set of deprecated keys to the global deprecations.
+
+ This method is lockless. It works by means of creating a new
+ DeprecationContext based on the old one, and then atomically swapping in
+ the new context. If someone else updated the context in between us reading
+ the old context and swapping in the new one, we try again until we win the
+ race.
+
+ @param deltas The deprecations to add.]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="use {@link #addDeprecation(String key, String newKey,
+ String customMessage)} instead">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKeys" type="java.lang.String[]"/>
+ <param name="customMessage" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If a key is deprecated in favor of multiple keys, they are all treated as
+ aliases of each other, and setting any one of them resets all the others
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key
+ @param newKeys
+ @param customMessage
+ @deprecated use {@link #addDeprecation(String key, String newKey,
+ String customMessage)} instead]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKey" type="java.lang.String"/>
+ <param name="customMessage" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key
+ @param newKey
+ @param customMessage]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="use {@link #addDeprecation(String key, String newKey)} instead">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKeys" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If a key is deprecated in favor of multiple keys, they are all treated as
+ aliases of each other, and setting any one of them resets all the others
+ to the new value.
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKeys list of keys that take up the values of deprecated key
+ @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]>
+ </doc>
+ </method>
+ <method name="addDeprecation"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.String"/>
+ <param name="newKey" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Adds the deprecated key to the global deprecation map when no custom
+ message is provided.
+ It does not override any existing entries in the deprecation map.
+ This is to be used only by the developers in order to add deprecation of
+ keys, and attempts to call this method after loading resources once,
+ would lead to <tt>UnsupportedOperationException</tt>
+
+ If you have multiple deprecation entries to add, it is more efficient to
+ use #addDeprecations(DeprecationDelta[] deltas) instead.
+
+ @param key Key that is to be deprecated
+ @param newKey key that takes up the value of deprecated key]]>
+ </doc>
+ </method>
+ <method name="isDeprecated" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="key" type="java.lang.String"/>
+ <doc>
+ <![CDATA[checks whether the given <code>key</code> is deprecated.
+
+ @param key the parameter which is to be checked for deprecation
+ @return <code>true</code> if the key is deprecated and
+ <code>false</code> otherwise.]]>
+ </doc>
+ </method>
+ <method name="setDeprecatedProperties"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Sets all deprecated properties that are not currently set but have a
+ corresponding new property that is set. Useful for iterating the
+ properties when all deprecated properties for currently set properties
+ need to be present.]]>
+ </doc>
+ </method>
+ <method name="addDefaultResource"
+ abstract="false" native="false" synchronized="true"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a default resource. Resources are loaded in the order of the resources
+ added.
+ @param name file name. File should be present in the classpath.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param name resource to be added, the classpath is examined for a file
+ with that name.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="url" type="java.net.URL"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param url url of the resource to be added, the local filesystem is
+ examined directly to find the resource, without referring to
+ the classpath.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="file" type="org.apache.hadoop.fs.Path"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param file file-path of resource to be added, the local filesystem is
+ examined directly to find the resource, without referring to
+ the classpath.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.InputStream"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ WARNING: The contents of the InputStream will be cached, by this method.
+ So use this sparingly because it does increase the memory consumption.
+
+ @param in InputStream to deserialize the object from. In will be read from
+ when a get or set is called next. After it is read the stream will be
+ closed.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.InputStream"/>
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param in InputStream to deserialize the object from.
+ @param name the name of the resource because InputStream.toString is not
+ very descriptive some times.]]>
+ </doc>
+ </method>
+ <method name="addResource"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ <doc>
+ <![CDATA[Add a configuration resource.
+
+ The properties of this resource will override properties of previously
+ added resources, unless they were marked <a href="#Final">final</a>.
+
+ @param conf Configuration object from which to load properties]]>
+ </doc>
+ </method>
+ <method name="reloadConfiguration"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Reload configuration from previously added resources.
+
+ This method will clear all the configuration read from the added
+ resources, and final parameters. This will make the resources to
+ be read again before accessing the values. Values that are added
+ via set methods will overlay values read from the resources.]]>
+ </doc>
+ </method>
+ <method name="get" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property, <code>null</code> if
+ no such property exists. If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null.
+
+ Values are processed for <a href="#VariableExpansion">variable expansion</a>
+ before being returned.
+
+ @param name the property name, will be trimmed before get value.
+ @return the value of the <code>name</code> or its replacing property,
+ or null if no such property exists.]]>
+ </doc>
+ </method>
+ <method name="setAllowNullValueProperties"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="val" type="boolean"/>
+ <doc>
+ <![CDATA[Set Configuration to allow keys without values during setup. Intended
+ for use during testing.
+
+ @param val If true, will allow Configuration to store keys without values]]>
+ </doc>
+ </method>
+ <method name="onlyKeyExists" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Return existence of the <code>name</code> property, but only for
+ names which have no valid value, usually non-existent or commented
+ out in XML.
+
+ @param name the property name
+ @return true if the property <code>name</code> exists without value]]>
+ </doc>
+ </method>
+ <method name="getTrimmed" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>,
+ <code>null</code> if no such property exists.
+ If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null
+
+ Values are processed for <a href="#VariableExpansion">variable expansion</a>
+ before being returned.
+
+ @param name the property name.
+ @return the value of the <code>name</code> or its replacing property,
+ or null if no such property exists.]]>
+ </doc>
+ </method>
+ <method name="getTrimmed" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a trimmed <code>String</code>,
+ <code>defaultValue</code> if no such property exists.
+ See @{Configuration#getTrimmed} for more details.
+
+ @param name the property name.
+ @param defaultValue the property default value.
+ @return the value of the <code>name</code> or defaultValue
+ if it is not set.]]>
+ </doc>
+ </method>
+ <method name="getRaw" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property, without doing
+ <a href="#VariableExpansion">variable expansion</a>.If the key is
+ deprecated, it returns the value of the first key which replaces
+ the deprecated key and is not null.
+
+ @param name the property name.
+ @return the value of the <code>name</code> property or
+ its replacing property and null if no such property exists.]]>
+ </doc>
+ </method>
+ <method name="set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the <code>value</code> of the <code>name</code> property. If
+ <code>name</code> is deprecated or there is a deprecated name associated to it,
+ it sets the value to both names. Name will be trimmed before put into
+ configuration.
+
+ @param name property name.
+ @param value property value.]]>
+ </doc>
+ </method>
+ <method name="set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="java.lang.String"/>
+ <param name="source" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Set the <code>value</code> of the <code>name</code> property. If
+ <code>name</code> is deprecated, it also sets the <code>value</code> to
+ the keys that replace the deprecated key. Name will be trimmed before put
+ into configuration.
+
+ @param name property name.
+ @param value property value.
+ @param source the place that this configuration value came from
+ (For debugging).
+ @throws IllegalArgumentException when the value or name is null.]]>
+ </doc>
+ </method>
+ <method name="unset"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Unset a previously set property.]]>
+ </doc>
+ </method>
+ <method name="setIfUnset"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Sets a property if it is currently unset.
+ @param name the property name
+ @param value the new value]]>
+ </doc>
+ </method>
+ <method name="get" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code>. If the key is deprecated,
+ it returns the value of the first key which replaces the deprecated key
+ and is not null.
+ If no such property exists,
+ then <code>defaultValue</code> is returned.
+
+ @param name property name, will be trimmed before get value.
+ @param defaultValue default value.
+ @return property value, or <code>defaultValue</code> if the property
+ doesn't exist.]]>
+ </doc>
+ </method>
+ <method name="getInt" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="int"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as an <code>int</code>.
+
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>int</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as an <code>int</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getInts" return="int[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a set of comma-delimited
+ <code>int</code> values.
+
+ If no such property exists, an empty array is returned.
+
+ @param name property name
+ @return property value interpreted as an array of comma-delimited
+ <code>int</code> values]]>
+ </doc>
+ </method>
+ <method name="setInt"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="int"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to an <code>int</code>.
+
+ @param name property name.
+ @param value <code>int</code> value of the property.]]>
+ </doc>
+ </method>
+ <method name="getLong" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="long"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>long</code>.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>long</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getLongBytes" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="long"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>long</code> or
+ human readable format. If no such property exists, the provided default
+ value is returned, or if the specified value is not a valid
+ <code>long</code> or human readable format, then an error is thrown. You
+ can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ t(tera), p(peta), e(exa)
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>long</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setLong"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="long"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>long</code>.
+
+ @param name property name.
+ @param value <code>long</code> value of the property.]]>
+ </doc>
+ </method>
+ <method name="getFloat" return="float"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="float"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>float</code>.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>float</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>float</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setFloat"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="float"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>float</code>.
+
+ @param name property name.
+ @param value property value.]]>
+ </doc>
+ </method>
+ <method name="getDouble" return="double"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="double"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>double</code>.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid <code>double</code>,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a <code>double</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setDouble"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="double"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>double</code>.
+
+ @param name property name.
+ @param value property value.]]>
+ </doc>
+ </method>
+ <method name="getBoolean" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="boolean"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>boolean</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>boolean</code>, then <code>defaultValue</code> is returned.
+
+ @param name property name.
+ @param defaultValue default value.
+ @return property value as a <code>boolean</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="setBoolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="boolean"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to a <code>boolean</code>.
+
+ @param name property name.
+ @param value <code>boolean</code> value of the property.]]>
+ </doc>
+ </method>
+ <method name="setBooleanIfUnset"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="boolean"/>
+ <doc>
+ <![CDATA[Set the given property, if it is currently unset.
+ @param name property name
+ @param value new value]]>
+ </doc>
+ </method>
+ <method name="setEnum"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="T"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to the given type. This
+ is equivalent to <code>set(<name>, value.toString())</code>.
+ @param name property name
+ @param value new value]]>
+ </doc>
+ </method>
+ <method name="getEnum" return="T"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="T"/>
+ <doc>
+ <![CDATA[Return value matching this enumerated type.
+ Note that the returned value is trimmed by this method.
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists
+ @throws IllegalArgumentException If mapping is illegal for the type
+ provided]]>
+ </doc>
+ </method>
+ <method name="setTimeDuration"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="value" type="long"/>
+ <param name="unit" type="java.util.concurrent.TimeUnit"/>
+ <doc>
+ <![CDATA[Set the value of <code>name</code> to the given time duration. This
+ is equivalent to <code>set(<name>, value + <time suffix>)</code>.
+ @param name Property name
+ @param value Time duration
+ @param unit Unit of time]]>
+ </doc>
+ </method>
+ <method name="getTimeDuration" return="long"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="long"/>
+ <param name="unit" type="java.util.concurrent.TimeUnit"/>
+ <doc>
+ <![CDATA[Return time duration in the given time unit. Valid units are encoded in
+ properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
+ (ms), seconds (s), minutes (m), hours (h), and days (d).
+ @param name Property name
+ @param defaultValue Value returned if no mapping exists.
+ @param unit Unit to convert the stored property, if it exists.
+ @throws NumberFormatException If the property stripped of its unit is not
+ a number]]>
+ </doc>
+ </method>
+ <method name="getPattern" return="java.util.regex.Pattern"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.util.regex.Pattern"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>Pattern</code>.
+ If no such property is specified, or if the specified value is not a valid
+ <code>Pattern</code>, then <code>DefaultValue</code> is returned.
+ Note that the returned value is NOT trimmed by this method.
+
+ @param name property name
+ @param defaultValue default value
+ @return property value as a compiled Pattern, or defaultValue]]>
+ </doc>
+ </method>
+ <method name="setPattern"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="pattern" type="java.util.regex.Pattern"/>
+ <doc>
+ <![CDATA[Set the given property to <code>Pattern</code>.
+ If the pattern is passed as null, sets the empty pattern which results in
+ further calls to getPattern(...) returning the default value.
+
+ @param name property name
+ @param pattern new value]]>
+ </doc>
+ </method>
+ <method name="getPropertySources" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Gets information about why a property was set. Typically this is the
+ path to the resource objects (file, URL, etc.) the property came from, but
+ it can also indicate that it was set programatically, or because of the
+ command line.
+
+ @param name - The property name to get the source of.
+ @return null - If the property or its source wasn't found. Otherwise,
+ returns a list of the sources of the resource. The older sources are
+ the first ones in the list. So for example if a configuration is set from
+ the command line, and then written out to a file that is read back in the
+ first entry would indicate that it was set from the command line, while
+ the second one would indicate the file that the new configuration was read
+ in from.]]>
+ </doc>
+ </method>
+ <method name="getRange" return="org.apache.hadoop.conf.Configuration.IntegerRanges"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Parse the given attribute as a set of integer ranges
+ @param name the attribute name
+ @param defaultValue the default value if it is not set
+ @return a new set of ranges from the configured value]]>
+ </doc>
+ </method>
+ <method name="getStringCollection" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ a collection of <code>String</code>s.
+ If no such property is specified then empty collection is returned.
+ <p>
+ This is an optimized version of {@link #getStrings(String)}
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s.]]>
+ </doc>
+ </method>
+ <method name="getStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s.
+ If no such property is specified then <code>null</code> is returned.
+
+ @param name property name.
+ @return property value as an array of <code>String</code>s,
+ or <code>null</code>.]]>
+ </doc>
+ </method>
+ <method name="getStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s.
+ If no such property is specified then default value is returned.
+
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of <code>String</code>s,
+ or default value.]]>
+ </doc>
+ </method>
+ <method name="getTrimmedStringCollection" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then empty <code>Collection</code> is returned.
+
+ @param name property name.
+ @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>]]>
+ </doc>
+ </method>
+ <method name="getTrimmedStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then an empty array is returned.
+
+ @param name property name.
+ @return property value as an array of trimmed <code>String</code>s,
+ or empty array.]]>
+ </doc>
+ </method>
+ <method name="getTrimmedStrings" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Get the comma delimited values of the <code>name</code> property as
+ an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
+ If no such property is specified then default value is returned.
+
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of trimmed <code>String</code>s,
+ or default value.]]>
+ </doc>
+ </method>
+ <method name="setStrings"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="values" type="java.lang.String[]"/>
+ <doc>
+ <![CDATA[Set the array of string values for the <code>name</code> property as
+ as comma delimited values.
+
+ @param name property name.
+ @param values The values]]>
+ </doc>
+ </method>
+ <method name="getPassword" return="char[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get the value for a known password configuration element.
+ In order to enable the elimination of clear text passwords in config,
+ this method attempts to resolve the property name as an alias through
+ the CredentialProvider API and conditionally fallsback to config.
+ @param name property name
+ @return password]]>
+ </doc>
+ </method>
+ <method name="getPasswordFromCredentialProviders" return="char[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Try and resolve the provided element name as a credential provider
+ alias.
+ @param name alias of the provisioned credential
+ @return password or null if not found
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getPasswordFromConfig" return="char[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Fallback to clear text passwords in configuration.
+ @param name
+ @return clear text password or null]]>
+ </doc>
+ </method>
+ <method name="getSocketAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="hostProperty" type="java.lang.String"/>
+ <param name="addressProperty" type="java.lang.String"/>
+ <param name="defaultAddressValue" type="java.lang.String"/>
+ <param name="defaultPort" type="int"/>
+ <doc>
+ <![CDATA[Get the socket address for <code>hostProperty</code> as a
+ <code>InetSocketAddress</code>. If <code>hostProperty</code> is
+ <code>null</code>, <code>addressProperty</code> will be used. This
+ is useful for cases where we want to differentiate between host
+ bind address and address clients should use to establish connection.
+
+ @param hostProperty bind host property name.
+ @param addressProperty address property name.
+ @param defaultAddressValue the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+ </doc>
+ </method>
+ <method name="getSocketAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultAddress" type="java.lang.String"/>
+ <param name="defaultPort" type="int"/>
+ <doc>
+ <![CDATA[Get the socket address for <code>name</code> property as a
+ <code>InetSocketAddress</code>.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+ </doc>
+ </method>
+ <method name="setSocketAddr"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <doc>
+ <![CDATA[Set the socket address for the <code>name</code> property as
+ a <code>host:port</code>.]]>
+ </doc>
+ </method>
+ <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="hostProperty" type="java.lang.String"/>
+ <param name="addressProperty" type="java.lang.String"/>
+ <param name="defaultAddressValue" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <doc>
+ <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>. The wildcard
+ address is replaced with the local host's address. If the host and address
+ properties are configured the host component of the address will be combined
+ with the port component of the addr to generate the address. This is to allow
+ optional control over which host name is used in multi-home bind-host
+ cases where a host can have multiple names
+ @param hostProperty the bind-host configuration name
+ @param addressProperty the service address configuration name
+ @param defaultAddressValue the service default address configuration value
+ @param addr InetSocketAddress of the service listener
+ @return InetSocketAddress for clients to connect]]>
+ </doc>
+ </method>
+ <method name="updateConnectAddr" return="java.net.InetSocketAddress"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="addr" type="java.net.InetSocketAddress"/>
+ <doc>
+ <![CDATA[Set the socket address a client can use to connect for the
+ <code>name</code> property as a <code>host:port</code>. The wildcard
+ address is replaced with the local host's address.
+ @param name property name.
+ @param addr InetSocketAddress of a listener to store in the given property
+ @return InetSocketAddress for clients to connect]]>
+ </doc>
+ </method>
+ <method name="getClassByName" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <exception name="ClassNotFoundException" type="java.lang.ClassNotFoundException"/>
+ <doc>
+ <![CDATA[Load a class by name.
+
+ @param name the class name.
+ @return the class object.
+ @throws ClassNotFoundException if the class is not found.]]>
+ </doc>
+ </method>
+ <method name="getClassByNameOrNull" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Load a class by name, returning null rather than throwing an exception
+ if it couldn't be loaded. This is to avoid the overhead of creating
+ an exception.
+
+ @param name the class name
+ @return the class object, or null if it could not be found.]]>
+ </doc>
+ </method>
+ <method name="getClasses" return="java.lang.Class[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.Class[]"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property
+ as an array of <code>Class</code>.
+ The value of the property specifies a list of comma separated class names.
+ If no such property is specified, then <code>defaultValue</code> is
+ returned.
+
+ @param name the property name.
+ @param defaultValue default value.
+ @return property value as a <code>Class[]</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>.
+ If no such property is specified, then <code>defaultValue</code> is
+ returned.
+
+ @param name the class name.
+ @param defaultValue default value.
+ @return property value as a <code>Class</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getClass" return="java.lang.Class"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="defaultValue" type="java.lang.Class"/>
+ <param name="xface" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>Class</code>
+ implementing the interface specified by <code>xface</code>.
+
+ If no such property is specified, then <code>defaultValue</code> is
+ returned.
+
+ An exception is thrown if the returned class does not implement the named
+ interface.
+
+ @param name the class name.
+ @param defaultValue default value.
+ @param xface the interface implemented by the named class.
+ @return property value as a <code>Class</code>,
+ or <code>defaultValue</code>.]]>
+ </doc>
+ </method>
+ <method name="getInstances" return="java.util.List"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="xface" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Get the value of the <code>name</code> property as a <code>List</code>
+ of objects implementing the interface specified by <code>xface</code>.
+
+ An exception is thrown if any of the classes does not exist, or if it does
+ not implement the named interface.
+
+ @param name the property name.
+ @param xface the interface implemented by the classes named by
+ <code>name</code>.
+ @return a <code>List</code> of objects implementing <code>xface</code>.]]>
+ </doc>
+ </method>
+ <method name="setClass"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <param name="theClass" type="java.lang.Class"/>
+ <param name="xface" type="java.lang.Class"/>
+ <doc>
+ <![CDATA[Set the value of the <code>name</code> property to the name of a
+ <code>theClass</code> implementing the given interface <code>xface</code>.
+
+ An exception is thrown if <code>theClass</code> does not implement the
+ interface <code>xface</code>.
+
+ @param name property name.
+ @param theClass property value.
+ @param xface the interface implemented by the named class.]]>
+ </doc>
+ </method>
+ <method name="getLocalPath" return="org.apache.hadoop.fs.Path"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="dirsProp" type="java.lang.String"/>
+ <param name="path" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get a local file under a directory named by <i>dirsProp</i> with
+ the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code. If the selected
+ directory does not exist, an attempt is made to create it.
+
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+ </doc>
+ </method>
+ <method name="getFile" return="java.io.File"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="dirsProp" type="java.lang.String"/>
+ <param name="path" type="java.lang.String"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Get a local file name under a directory named in <i>dirsProp</i> with
+ the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
+ then one is chosen based on <i>path</i>'s hash code. If the selected
+ directory does not exist, an attempt is made to create it.
+
+ @param dirsProp directory in which to locate the file.
+ @param path file-path.
+ @return local file under the directory with the given path.]]>
+ </doc>
+ </method>
+ <method name="getResource" return="java.net.URL"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get the {@link URL} for the named resource.
+
+ @param name resource name.
+ @return the url for the named resource.]]>
+ </doc>
+ </method>
+ <method name="getConfResourceAsInputStream" return="java.io.InputStream"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get an input stream attached to the configuration resource with the
+ given <code>name</code>.
+
+ @param name configuration resource name.
+ @return an input stream attached to the resource.]]>
+ </doc>
+ </method>
+ <method name="getConfResourceAsReader" return="java.io.Reader"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Get a {@link Reader} attached to the configuration resource with the
+ given <code>name</code>.
+
+ @param name configuration resource name.
+ @return a reader attached to the resource.]]>
+ </doc>
+ </method>
+ <method name="getFinalParameters" return="java.util.Set"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the set of parameters marked final.
+
+ @return final parameter set.]]>
+ </doc>
+ </method>
+ <method name="getProps" return="java.util.Properties"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ </method>
+ <method name="size" return="int"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return the number of keys in the configuration.
+
+ @return number of keys in the configuration.]]>
+ </doc>
+ </method>
+ <method name="clear"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Clears all keys from the configuration.]]>
+ </doc>
+ </method>
+ <method name="iterator" return="java.util.Iterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get an {@link Iterator} to go through the list of <code>String</code>
+ key-value pairs in the configuration.
+
+ @return an iterator over the entries.]]>
+ </doc>
+ </method>
+ <method name="writeXml"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.OutputStream"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link OutputStream} using UTF-8 encoding.
+
+ @param out the output stream to write to.]]>
+ </doc>
+ </method>
+ <method name="writeXml"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.Writer"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Write out the non-default properties in this configuration to the given
+ {@link Writer}.
+
+ @param out the writer to write to.]]>
+ </doc>
+ </method>
+ <method name="dumpConfiguration"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="config" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="out" type="java.io.Writer"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Writes out all the parameters and their properties (final and resource) to
+ the given {@link Writer}
+ The format of the output would be
+ { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
+ key2.isFinal,key2.resource}... ] }
+ It does not output the parameters of the configuration object which is
+ loaded from an input stream.
+ @param out the Writer to write to
+ @throws IOException]]>
+ </doc>
+ </method>
+ <method name="getClassLoader" return="java.lang.ClassLoader"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get the {@link ClassLoader} for this job.
+
+ @return the correct class loader.]]>
+ </doc>
+ </method>
+ <method name="setClassLoader"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="classLoader" type="java.lang.ClassLoader"/>
+ <doc>
+ <![CDATA[Set the class loader that will be used to load the various objects.
+
+ @param classLoader the new class loader.]]>
+ </doc>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="setQuietMode"
+ abstract="false" native="false" synchronized="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="quietmode" type="boolean"/>
+ <doc>
+ <![CDATA[Set the quietness-mode.
+
+ In the quiet-mode, error and informational messages might not be logged.
+
+ @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
+ to turn it off.]]>
+ </doc>
+ </method>
+ <method name="main"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="args" type="java.lang.String[]"/>
+ <exception name="Exception" type="java.lang.Exception"/>
+ <doc>
+ <![CDATA[For debugging. List non-default properties to the terminal and exit.]]>
+ </doc>
+ </method>
+ <method name="readFields"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="in" type="java.io.DataInput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="write"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="out" type="java.io.DataOutput"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="getValByRegex" return="java.util.Map"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="regex" type="java.lang.String"/>
+ <doc>
+ <![CDATA[get keys matching the the regex
+ @param regex
+ @return Map<String,String> with matching keys]]>
+ </doc>
+ </method>
+ <method name="dumpDeprecatedKeys"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="hasWarnedDeprecation" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="name" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Returns whether or not a deprecated name has been warned. If the name is not
+ deprecated then always return false]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Provides access to configuration parameters.
+
+ <h4 id="Resources">Resources</h4>
+
+ <p>Configurations are specified by resources. A resource contains a set of
+ name/value pairs as XML data. Each resource is named by either a
+ <code>String</code> or by a {@link Path}. If named by a <code>String</code>,
+ then the classpath is examined for a file with that name. If named by a
+ <code>Path</code>, then the local filesystem is examined directly, without
+ referring to the classpath.
+
+ <p>Unless explicitly turned off, Hadoop by default specifies two
+ resources, loaded in-order from the classpath: <ol>
+ <li><tt>
+ <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ core-default.xml</a></tt>: Read-only defaults for hadoop.</li>
+ <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
+ installation.</li>
+ </ol>
+ Applications may add additional resources, which are loaded
+ subsequent to these resources in the order they are added.
+
+ <h4 id="FinalParams">Final Parameters</h4>
+
+ <p>Configuration parameters may be declared <i>final</i>.
+ Once a resource declares a value final, no subsequently-loaded
+ resource can alter that value.
+ For example, one might define a final parameter with:
+ <tt><pre>
+ <property>
+ <name>dfs.hosts.include</name>
+ <value>/etc/hadoop/conf/hosts.include</value>
+ <b><final>true</final></b>
+ </property></pre></tt>
+
+ Administrators typically define parameters as final in
+ <tt>core-site.xml</tt> for values that user applications may not alter.
+
+ <h4 id="VariableExpansion">Variable Expansion</h4>
+
+ <p>Value strings are first processed for <i>variable expansion</i>. The
+ available properties are:<ol>
+ <li>Other properties defined in this Configuration; and, if a name is
+ undefined here,</li>
+ <li>Properties in {@link System#getProperties()}.</li>
+ </ol>
+
+ <p>For example, if a configuration resource contains the following property
+ definitions:
+ <tt><pre>
+ <property>
+ <name>basedir</name>
+ <value>/user/${<i>user.name</i>}</value>
+ </property>
+
+ <property>
+ <name>tempdir</name>
+ <value>${<i>basedir</i>}/tmp</value>
+ </property></pre></tt>
+
+ When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
+ will be resolved to another property in this Configuration, while
+ <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
+ of the System property with that name.
+ By default, warnings will be given to any deprecated configuration
+ parameters and these are suppressible by configuring
+ <tt>log4j.logger.org.apache.hadoop.conf.Configuration.deprecation</tt> in
+ log4j.properties file.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configuration -->
+ <!-- start class org.apache.hadoop.conf.Configuration.DeprecationDelta -->
+ <class name="Configuration.DeprecationDelta" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="DeprecationDelta" type="java.lang.String, java.lang.String, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="DeprecationDelta" type="java.lang.String, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="getKey" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getNewKeys" return="java.lang.String[]"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="getCustomMessage" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[A pending addition to the global set of deprecated keys.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configuration.DeprecationDelta -->
+ <!-- start class org.apache.hadoop.conf.Configuration.IntegerRanges -->
+ <class name="Configuration.IntegerRanges" extends="java.lang.Object"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="java.lang.Iterable"/>
+ <constructor name="IntegerRanges"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <constructor name="IntegerRanges" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="isIncluded" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="value" type="int"/>
+ <doc>
+ <![CDATA[Is the given value in the set of ranges
+ @param value the value to check
+ @return is the value in the ranges?]]>
+ </doc>
+ </method>
+ <method name="isEmpty" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[@return true if there are no values in this range, else false.]]>
+ </doc>
+ </method>
+ <method name="toString" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="iterator" return="java.util.Iterator"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[A class that represents a set of positive integer ranges. It parses
+ strings of the form: "2-3,5,7-" where ranges are separated by comma and
+ the lower/upper bounds are separated by dash. Either the lower or upper
+ bound may be omitted meaning all values up to or over. So the string
+ above means 2, 3, 5, and 7, 8, 9, ...]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configuration.IntegerRanges -->
+ <!-- start class org.apache.hadoop.conf.Configured -->
+ <class name="Configured" extends="java.lang.Object"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.conf.Configurable"/>
+ <constructor name="Configured"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a Configured.]]>
+ </doc>
+ </constructor>
+ <constructor name="Configured" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a Configured.]]>
+ </doc>
+ </constructor>
+ <method name="setConf"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="getConf" return="org.apache.hadoop.conf.Configuration"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <doc>
+ <![CDATA[Base class for things that may be configured with a {@link Configuration}.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.Configured -->
+ <!-- start class org.apache.hadoop.conf.ConfServlet.BadFormatException -->
+ <class name="ConfServlet.BadFormatException" extends="java.lang.Exception"
+ abstract="false"
+ static="true" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="BadFormatException" type="java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ConfServlet.BadFormatException -->
+ <!-- start interface org.apache.hadoop.conf.Reconfigurable -->
+ <interface name="Reconfigurable" abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.conf.Configurable"/>
+ <method name="reconfigureProperty" return="java.lang.String"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <param name="newVal" type="java.lang.String"/>
+ <exception name="ReconfigurationException" type="org.apache.hadoop.conf.ReconfigurationException"/>
+ <doc>
+ <![CDATA[Change a configuration property on this object to the value specified.
+
+ Change a configuration property on this object to the value specified
+ and return the previous value that the configuration property was set to
+ (or null if it was not previously set). If newVal is null, set the property
+ to its default value;
+
+ If the property cannot be changed, throw a
+ {@link ReconfigurationException}.]]>
+ </doc>
+ </method>
+ <method name="isPropertyReconfigurable" return="boolean"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <doc>
+ <![CDATA[Return whether a given property is changeable at run time.
+
+ If isPropertyReconfigurable returns true for a property,
+ then changeConf should not throw an exception when changing
+ this property.]]>
+ </doc>
+ </method>
+ <method name="getReconfigurableProperties" return="java.util.Collection"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Return all the properties that can be changed at run time.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Something whose {@link Configuration} can be changed at run time.]]>
+ </doc>
+ </interface>
+ <!-- end interface org.apache.hadoop.conf.Reconfigurable -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurableBase -->
+ <class name="ReconfigurableBase" extends="org.apache.hadoop.conf.Configured"
+ abstract="true"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <implements name="org.apache.hadoop.conf.Reconfigurable"/>
+ <constructor name="ReconfigurableBase"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a ReconfigurableBase.]]>
+ </doc>
+ </constructor>
+ <constructor name="ReconfigurableBase" type="org.apache.hadoop.conf.Configuration"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Construct a ReconfigurableBase with the {@link Configuration}
+ conf.]]>
+ </doc>
+ </constructor>
+ <method name="setReconfigurationUtil"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="ru" type="org.apache.hadoop.conf.ReconfigurationUtil"/>
+ </method>
+ <method name="getChangedProperties" return="java.util.Collection"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="newConf" type="org.apache.hadoop.conf.Configuration"/>
+ <param name="oldConf" type="org.apache.hadoop.conf.Configuration"/>
+ </method>
+ <method name="startReconfigurationTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="IOException" type="java.io.IOException"/>
+ <doc>
+ <![CDATA[Start a reconfiguration task to reload configuration in background.]]>
+ </doc>
+ </method>
+ <method name="getReconfigurationTaskStatus" return="org.apache.hadoop.conf.ReconfigurationTaskStatus"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="shutdownReconfigurationTask"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </method>
+ <method name="reconfigureProperty" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="true" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <param name="newVal" type="java.lang.String"/>
+ <exception name="ReconfigurationException" type="org.apache.hadoop.conf.ReconfigurationException"/>
+ <doc>
+ <![CDATA[{@inheritDoc}
+
+ This method makes the change to this objects {@link Configuration}
+ and calls reconfigurePropertyImpl to update internal data structures.
+ This method cannot be overridden, subclasses should instead override
+ reconfigureProperty.]]>
+ </doc>
+ </method>
+ <method name="getReconfigurableProperties" return="java.util.Collection"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[{@inheritDoc}
+
+ Subclasses must override this.]]>
+ </doc>
+ </method>
+ <method name="isPropertyReconfigurable" return="boolean"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <doc>
+ <![CDATA[{@inheritDoc}
+
+ Subclasses may wish to override this with a more efficient implementation.]]>
+ </doc>
+ </method>
+ <method name="reconfigurePropertyImpl"
+ abstract="true" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="property" type="java.lang.String"/>
+ <param name="newVal" type="java.lang.String"/>
+ <exception name="ReconfigurationException" type="org.apache.hadoop.conf.ReconfigurationException"/>
+ <doc>
+ <![CDATA[Change a configuration property.
+
+ Subclasses must override this. This method applies the change to
+ all internal data structures derived from the configuration property
+ that is being changed. If this object owns other Reconfigurable objects
+ reconfigureProperty should be called recursively to make sure that
+ to make sure that the configuration of these objects is updated.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Utility base class for implementing the Reconfigurable interface.
+
+ Subclasses should override reconfigurePropertyImpl to change individual
+ properties and getReconfigurableProperties to get all properties that
+ can be changed at run time.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ReconfigurableBase -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurationException -->
+ <class name="ReconfigurationException" extends="java.lang.Exception"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ReconfigurationException"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Create a new instance of {@link ReconfigurationException}.]]>
+ </doc>
+ </constructor>
+ <constructor name="ReconfigurationException" type="java.lang.String, java.lang.String, java.lang.String, java.lang.Throwable"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Create a new instance of {@link ReconfigurationException}.]]>
+ </doc>
+ </constructor>
+ <constructor name="ReconfigurationException" type="java.lang.String, java.lang.String, java.lang.String"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Create a new instance of {@link ReconfigurationException}.]]>
+ </doc>
+ </constructor>
+ <method name="getProperty" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get property that cannot be changed.]]>
+ </doc>
+ </method>
+ <method name="getNewValue" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get value to which property was supposed to be changed.]]>
+ </doc>
+ </method>
+ <method name="getOldValue" return="java.lang.String"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <doc>
+ <![CDATA[Get old value of property that cannot be changed.]]>
+ </doc>
+ </method>
+ <doc>
+ <![CDATA[Exception indicating that configuration property cannot be changed
+ at run time.]]>
+ </doc>
+ </class>
+ <!-- end class org.apache.hadoop.conf.ReconfigurationException -->
+ <!-- start class org.apache.hadoop.conf.ReconfigurationServlet -->
+ <class name="ReconfigurationServlet" extends="javax.servlet.http.HttpServlet"
+ abstract="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <constructor name="ReconfigurationServlet"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ </constructor>
+ <method name="init"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="public"
+ deprecated="not deprecated">
+ <exception name="ServletException" type="javax.servlet.ServletException"/>
+ </method>
+ <method name="doGet"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="req" type="javax.servlet.http.HttpServletRequest"/>
+ <param name="resp" type="javax.servlet.http.HttpServletResponse"/>
+ <exception name="ServletException" type="javax.servlet.ServletException"/>
+ <exception name="IOException" type="java.io.IOException"/>
+ </method>
+ <method name="doPost"
+ abstract="false" native="false" synchronized="false"
+ static="false" final="false" visibility="protected"
+ deprecated="not deprecated">
+ <param name="req" type="javax.servlet.http.HttpServletRequest"/>
+ <param name="resp" type="javax.servlet.http.HttpSer
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/52] [abbrv] hadoop git commit: HADOOP-13524. mvn eclipse:eclipse
generates .gitignore'able files. Contributed by Vinod Kumar Vavilapalli
Posted by aw...@apache.org.
HADOOP-13524. mvn eclipse:eclipse generates .gitignore'able files. Contributed by Vinod Kumar Vavilapalli
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd76238a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd76238a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd76238a
Branch: refs/heads/HADOOP-13341
Commit: dd76238a3bafd58faa6f38f075505bef1012f150
Parents: e90f335
Author: Jian He <ji...@apache.org>
Authored: Tue Aug 23 19:13:14 2016 +0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Aug 23 19:13:14 2016 +0800
----------------------------------------------------------------------
.gitignore | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd76238a/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 998287d..a5d69d0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,11 @@
.settings
target
build
+
+# External tool builders
+*/.externalToolBuilders
+*/maven-eclipse.xml
+
hadoop-common-project/hadoop-kms/downloads/
hadoop-hdfs-project/hadoop-hdfs/downloads
hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/52] [abbrv] hadoop git commit: HDFS-8986. Add option to -du to
calculate directory space usage excluding snapshots. Contributed by Xiao
Chen.
Posted by aw...@apache.org.
HDFS-8986. Add option to -du to calculate directory space usage excluding snapshots. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0efea49
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0efea49
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0efea49
Branch: refs/heads/HADOOP-13341
Commit: f0efea490e5aa9dd629d2199aae9c5b1290a17ee
Parents: dd76238
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Tue Aug 23 04:13:48 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Tue Aug 23 04:14:24 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/ContentSummary.java | 127 ++++++++--
.../java/org/apache/hadoop/fs/shell/Count.java | 20 +-
.../org/apache/hadoop/fs/shell/FsUsage.java | 31 ++-
.../src/site/markdown/FileSystemShell.md | 11 +-
.../org/apache/hadoop/fs/shell/TestCount.java | 5 +-
.../src/test/resources/testConf.xml | 12 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 8 +
.../src/main/proto/hdfs.proto | 4 +
.../ContentSummaryComputationContext.java | 6 +
.../hadoop/hdfs/server/namenode/INode.java | 9 +-
.../hdfs/server/namenode/INodeDirectory.java | 4 +
.../org/apache/hadoop/hdfs/TestDFSShell.java | 235 ++++++++++++++++++-
12 files changed, 432 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index 3dedbcc..3e75951 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -34,6 +34,11 @@ public class ContentSummary extends QuotaUsage implements Writable{
private long length;
private long fileCount;
private long directoryCount;
+ // These fields are to track the snapshot-related portion of the values.
+ private long snapshotLength;
+ private long snapshotFileCount;
+ private long snapshotDirectoryCount;
+ private long snapshotSpaceConsumed;
/** We don't use generics. Instead override spaceConsumed and other methods
in order to keep backward compatibility. */
@@ -56,6 +61,26 @@ public class ContentSummary extends QuotaUsage implements Writable{
return this;
}
+ public Builder snapshotLength(long snapshotLength) {
+ this.snapshotLength = snapshotLength;
+ return this;
+ }
+
+ public Builder snapshotFileCount(long snapshotFileCount) {
+ this.snapshotFileCount = snapshotFileCount;
+ return this;
+ }
+
+ public Builder snapshotDirectoryCount(long snapshotDirectoryCount) {
+ this.snapshotDirectoryCount = snapshotDirectoryCount;
+ return this;
+ }
+
+ public Builder snapshotSpaceConsumed(long snapshotSpaceConsumed) {
+ this.snapshotSpaceConsumed = snapshotSpaceConsumed;
+ return this;
+ }
+
@Override
public Builder quota(long quota){
super.quota(quota);
@@ -107,6 +132,10 @@ public class ContentSummary extends QuotaUsage implements Writable{
private long length;
private long fileCount;
private long directoryCount;
+ private long snapshotLength;
+ private long snapshotFileCount;
+ private long snapshotDirectoryCount;
+ private long snapshotSpaceConsumed;
}
/** Constructor deprecated by ContentSummary.Builder*/
@@ -142,17 +171,37 @@ public class ContentSummary extends QuotaUsage implements Writable{
this.length = builder.length;
this.fileCount = builder.fileCount;
this.directoryCount = builder.directoryCount;
+ this.snapshotLength = builder.snapshotLength;
+ this.snapshotFileCount = builder.snapshotFileCount;
+ this.snapshotDirectoryCount = builder.snapshotDirectoryCount;
+ this.snapshotSpaceConsumed = builder.snapshotSpaceConsumed;
}
/** @return the length */
public long getLength() {return length;}
+ public long getSnapshotLength() {
+ return snapshotLength;
+ }
+
/** @return the directory count */
public long getDirectoryCount() {return directoryCount;}
+ public long getSnapshotDirectoryCount() {
+ return snapshotDirectoryCount;
+ }
+
/** @return the file count */
public long getFileCount() {return fileCount;}
+ public long getSnapshotFileCount() {
+ return snapshotFileCount;
+ }
+
+ public long getSnapshotSpaceConsumed() {
+ return snapshotSpaceConsumed;
+ }
+
@Override
@InterfaceAudience.Private
public void write(DataOutput out) throws IOException {
@@ -180,9 +229,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
if (this == to) {
return true;
} else if (to instanceof ContentSummary) {
- return getLength() == ((ContentSummary) to).getLength() &&
- getFileCount() == ((ContentSummary) to).getFileCount() &&
- getDirectoryCount() == ((ContentSummary) to).getDirectoryCount() &&
+ ContentSummary right = (ContentSummary) to;
+ return getLength() == right.getLength() &&
+ getFileCount() == right.getFileCount() &&
+ getDirectoryCount() == right.getDirectoryCount() &&
+ getSnapshotLength() == right.getSnapshotLength() &&
+ getSnapshotFileCount() == right.getSnapshotFileCount() &&
+ getSnapshotDirectoryCount() == right.getSnapshotDirectoryCount() &&
+ getSnapshotSpaceConsumed() == right.getSnapshotSpaceConsumed() &&
super.equals(to);
} else {
return super.equals(to);
@@ -191,7 +245,9 @@ public class ContentSummary extends QuotaUsage implements Writable{
@Override
public int hashCode() {
- long result = getLength() ^ getFileCount() ^ getDirectoryCount();
+ long result = getLength() ^ getFileCount() ^ getDirectoryCount()
+ ^ getSnapshotLength() ^ getSnapshotFileCount()
+ ^ getSnapshotDirectoryCount() ^ getSnapshotSpaceConsumed();
return ((int) result) ^ super.hashCode();
}
@@ -255,15 +311,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
* @param qOption a flag indicating if quota needs to be printed or not
* @return the string representation of the object
*/
+ @Override
public String toString(boolean qOption) {
return toString(qOption, false);
}
/** Return the string representation of the object in the output format.
- * if qOption is false, output directory count, file count, and content size;
- * if qOption is true, output quota and remaining quota as well.
- * if hOption is false file sizes are returned in bytes
- * if hOption is true file sizes are returned in human readable
+ * For description of the options,
+ * @see #toString(boolean, boolean, boolean, boolean, List)
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output if to be used
@@ -273,10 +328,24 @@ public class ContentSummary extends QuotaUsage implements Writable{
return toString(qOption, hOption, false, null);
}
+ /** Return the string representation of the object in the output format.
+ * For description of the options,
+ * @see #toString(boolean, boolean, boolean, boolean, List)
+ *
+ * @param qOption a flag indicating if quota needs to be printed or not
+ * @param hOption a flag indicating if human readable output is to be used
+ * @param xOption a flag indicating if calculation from snapshots is to be
+ * included in the output
+ * @return the string representation of the object
+ */
+ public String toString(boolean qOption, boolean hOption, boolean xOption) {
+ return toString(qOption, hOption, false, xOption, null);
+ }
+
/**
* Return the string representation of the object in the output format.
- * if tOption is true, display the quota by storage types,
- * Otherwise, same logic with #toString(boolean,boolean)
+ * For description of the options,
+ * @see #toString(boolean, boolean, boolean, boolean, List)
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output if to be used
@@ -286,6 +355,29 @@ public class ContentSummary extends QuotaUsage implements Writable{
*/
public String toString(boolean qOption, boolean hOption,
boolean tOption, List<StorageType> types) {
+ return toString(qOption, hOption, tOption, false, types);
+ }
+
+ /** Return the string representation of the object in the output format.
+ * if qOption is false, output directory count, file count, and content size;
+ * if qOption is true, output quota and remaining quota as well.
+ * if hOption is false, file sizes are returned in bytes
+ * if hOption is true, file sizes are returned in human readable
+ * if tOption is true, display the quota by storage types
+ * if tOption is false, same logic with #toString(boolean,boolean)
+ * if xOption is false, output includes the calculation from snapshots
+ * if xOption is true, output excludes the calculation from snapshots
+ *
+ * @param qOption a flag indicating if quota needs to be printed or not
+ * @param hOption a flag indicating if human readable output is to be used
+ * @param tOption a flag indicating if display quota by storage types
+ * @param xOption a flag indicating if calculation from snapshots is to be
+ * included in the output
+ * @param types Storage types to display
+ * @return the string representation of the object
+ */
+ public String toString(boolean qOption, boolean hOption, boolean tOption,
+ boolean xOption, List<StorageType> types) {
String prefix = "";
if (tOption) {
@@ -296,10 +388,17 @@ public class ContentSummary extends QuotaUsage implements Writable{
prefix = getQuotaUsage(hOption);
}
- return prefix + String.format(SUMMARY_FORMAT,
- formatSize(directoryCount, hOption),
- formatSize(fileCount, hOption),
- formatSize(length, hOption));
+ if (xOption) {
+ return prefix + String.format(SUMMARY_FORMAT,
+ formatSize(directoryCount - snapshotDirectoryCount, hOption),
+ formatSize(fileCount - snapshotFileCount, hOption),
+ formatSize(length - snapshotLength, hOption));
+ } else {
+ return prefix + String.format(SUMMARY_FORMAT,
+ formatSize(directoryCount, hOption),
+ formatSize(fileCount, hOption),
+ formatSize(length, hOption));
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index 2b5a4cc..16f52e3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -51,6 +51,8 @@ public class Count extends FsCommand {
private static final String OPTION_HUMAN = "h";
private static final String OPTION_HEADER = "v";
private static final String OPTION_TYPE = "t";
+ // exclude snapshots from calculation. Only work on default columns.
+ private static final String OPTION_EXCLUDE_SNAPSHOT = "x";
//return the quota, namespace count and disk space usage.
private static final String OPTION_QUOTA_AND_USAGE = "u";
@@ -58,7 +60,8 @@ public class Count extends FsCommand {
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
- OPTION_QUOTA_AND_USAGE + "] <path> ...";
+ OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
+ + "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" +
@@ -72,6 +75,8 @@ public class Count extends FsCommand {
"The -" + OPTION_HUMAN +
" option shows file sizes in human readable format.\n" +
"The -" + OPTION_HEADER + " option displays a header line.\n" +
+ "The -" + OPTION_EXCLUDE_SNAPSHOT + " option excludes snapshots " +
+ "from being calculated. \n" +
"The -" + OPTION_TYPE + " option displays quota by storage types.\n" +
"It must be used with -" + OPTION_QUOTA + " option.\n" +
"If a comma-separated list of storage types is given after the -" +
@@ -87,6 +92,7 @@ public class Count extends FsCommand {
private boolean showQuotabyType;
private List<StorageType> storageTypes = null;
private boolean showQuotasAndUsageOnly;
+ private boolean excludeSnapshots;
/** Constructor */
public Count() {}
@@ -106,7 +112,8 @@ public class Count extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
- OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE);
+ OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
+ OPTION_EXCLUDE_SNAPSHOT);
cf.addOptionWithValue(OPTION_TYPE);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
@@ -115,6 +122,7 @@ public class Count extends FsCommand {
showQuotas = cf.getOpt(OPTION_QUOTA);
humanReadable = cf.getOpt(OPTION_HUMAN);
showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
+ excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
if (showQuotas || showQuotasAndUsageOnly) {
String types = cf.getOptValue(OPTION_TYPE);
@@ -125,6 +133,11 @@ public class Count extends FsCommand {
} else {
showQuotabyType = false;
}
+ if (excludeSnapshots) {
+ out.println(OPTION_QUOTA + " or " + OPTION_QUOTA_AND_USAGE + " option "
+ + "is given, the -x option is ignored.");
+ excludeSnapshots = false;
+ }
}
if (cf.getOpt(OPTION_HEADER)) {
@@ -163,7 +176,8 @@ public class Count extends FsCommand {
storageTypes) + src);
} else {
ContentSummary summary = src.fs.getContentSummary(src.path);
- out.println(summary.toString(showQuotas, isHumanReadable()) + src);
+ out.println(summary.
+ toString(showQuotas, isHumanReadable(), excludeSnapshots) + src);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
index 765b181..197920f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
@@ -107,27 +107,30 @@ class FsUsage extends FsCommand {
/** show disk usage */
public static class Du extends FsUsage {
public static final String NAME = "du";
- public static final String USAGE = "[-s] [-h] <path> ...";
+ public static final String USAGE = "[-s] [-h] [-x] <path> ...";
public static final String DESCRIPTION =
- "Show the amount of space, in bytes, used by the files that " +
- "match the specified file pattern. The following flags are optional:\n" +
- "-s: Rather than showing the size of each individual file that" +
- " matches the pattern, shows the total (summary) size.\n" +
- "-h: Formats the sizes of files in a human-readable fashion" +
- " rather than a number of bytes.\n\n" +
- "Note that, even without the -s option, this only shows size summaries " +
- "one level deep into a directory.\n\n" +
- "The output is in the form \n" +
- "\tsize\tdisk space consumed\tname(full path)\n";
+ "Show the amount of space, in bytes, used by the files that match " +
+ "the specified file pattern. The following flags are optional:\n" +
+ "-s: Rather than showing the size of each individual file that" +
+ " matches the pattern, shows the total (summary) size.\n" +
+ "-h: Formats the sizes of files in a human-readable fashion" +
+ " rather than a number of bytes.\n" +
+ "-x: Excludes snapshots from being counted.\n\n" +
+ "Note that, even without the -s option, this only shows size " +
+ "summaries one level deep into a directory.\n\n" +
+ "The output is in the form \n" +
+ "\tsize\tdisk space consumed\tname(full path)\n";
protected boolean summary = false;
+ private boolean excludeSnapshots = false;
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
- CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s");
+ CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "x");
cf.parse(args);
humanReadable = cf.getOpt("h");
summary = cf.getOpt("s");
+ excludeSnapshots = cf.getOpt("x");
if (args.isEmpty()) args.add(Path.CUR_DIR);
}
@@ -156,6 +159,10 @@ class FsUsage extends FsCommand {
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
long length = contentSummary.getLength();
long spaceConsumed = contentSummary.getSpaceConsumed();
+ if (excludeSnapshots) {
+ length -= contentSummary.getSnapshotLength();
+ spaceConsumed -= contentSummary.getSnapshotSpaceConsumed();
+ }
usagesTable.addRow(formatSize(length), formatSize(spaceConsumed), item);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 066cfe3..a8a1920 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -132,10 +132,12 @@ Similar to get command, except that the destination is restricted to a local fil
count
-----
-Usage: `hadoop fs -count [-q] [-h] [-v] [-t [<storage type>]] [-u] <paths> `
+Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] <paths> `
Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+The -u and -q options control what columns the output contains. -q means show quotas, -u limits the output to show quotas and usage only.
+
The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA
@@ -146,6 +148,8 @@ The -h option shows sizes in human readable format.
The -v option displays a header line.
+The -x option excludes snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path. The -x option is ignored if -u or -q option is given.
+
Example:
* `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@@ -211,14 +215,15 @@ Example:
du
----
-Usage: `hadoop fs -du [-s] [-h] URI [URI ...]`
+Usage: `hadoop fs -du [-s] [-h] [-x] URI [URI ...]`
Displays sizes of files and directories contained in the given directory or the length of a file in case its just a file.
Options:
-* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files.
+* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. Without the -s option, calculation is done by going 1-level deep from the given path.
* The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864)
+* The -x option will exclude snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path.
The du returns three columns with the following format:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
index 04174e0..c7b30e4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
@@ -447,7 +447,7 @@ public class TestCount {
Count count = new Count();
String actual = count.getUsage();
String expected =
- "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] <path> ...";
+ "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
@@ -465,6 +465,7 @@ public class TestCount {
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "The -h option shows file sizes in human readable format.\n"
+ "The -v option displays a header line.\n"
+ + "The -x option excludes snapshots from being calculated. \n"
+ "The -t option displays quota by storage types.\n"
+ "It must be used with -q option.\n"
+ "If a comma-separated list of storage types is given after the -t option, \n"
@@ -521,7 +522,7 @@ public class TestCount {
}
@Override
- public String toString(boolean qOption, boolean hOption) {
+ public String toString(boolean qOption, boolean hOption, boolean xOption) {
if (qOption) {
if (hOption) {
return (HUMAN + WITH_QUOTAS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 82bc789..a31c828 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -200,7 +200,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-du \[-s\] \[-h\] <path> \.\.\. :\s*</expected-output>
+ <expected-output>^-du \[-s\] \[-h\] \[-x\] <path> \.\.\. :\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -228,6 +228,10 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
+ <expected-output>^\s*-x\s*Excludes snapshots from being counted.\s*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
<expected-output>^\s*Note that, even without the -s option, this only shows size summaries one level\s*</expected-output>
</comparator>
<comparator>
@@ -274,7 +278,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] <path> \.\.\. :( )*</expected-output>
+ <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[<storage type>\]\] \[-u\] \[-x\] <path> \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -308,6 +312,10 @@
<type>RegexpComparator</type>
<expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
</comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*The -x option excludes snapshots from being calculated.( )*</expected-output>
+ </comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 3c5f583..7df68e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -1466,6 +1466,10 @@ public class PBHelperClient {
builder.length(cs.getLength()).
fileCount(cs.getFileCount()).
directoryCount(cs.getDirectoryCount()).
+ snapshotLength(cs.getSnapshotLength()).
+ snapshotFileCount(cs.getSnapshotFileCount()).
+ snapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
+ snapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
quota(cs.getQuota()).
spaceConsumed(cs.getSpaceConsumed()).
spaceQuota(cs.getSpaceQuota());
@@ -2069,6 +2073,10 @@ public class PBHelperClient {
builder.setLength(cs.getLength()).
setFileCount(cs.getFileCount()).
setDirectoryCount(cs.getDirectoryCount()).
+ setSnapshotLength(cs.getSnapshotLength()).
+ setSnapshotFileCount(cs.getSnapshotFileCount()).
+ setSnapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
+ setSnapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
setQuota(cs.getQuota()).
setSpaceConsumed(cs.getSpaceConsumed()).
setSpaceQuota(cs.getSpaceQuota());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index f939ec3..7de8550 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -138,6 +138,10 @@ message ContentSummaryProto {
required uint64 spaceConsumed = 5;
required uint64 spaceQuota = 6;
optional StorageTypeQuotaInfosProto typeQuotaInfos = 7;
+ optional uint64 snapshotLength = 8;
+ optional uint64 snapshotFileCount = 9;
+ optional uint64 snapshotDirectoryCount = 10;
+ optional uint64 snapshotSpaceConsumed = 11;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
index 5739835..6df9e75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
@@ -29,6 +29,7 @@ public class ContentSummaryComputationContext {
private FSNamesystem fsn = null;
private BlockStoragePolicySuite bsps = null;
private ContentCounts counts = null;
+ private ContentCounts snapshotCounts = null;
private long nextCountLimit = 0;
private long limitPerRun = 0;
private long yieldCount = 0;
@@ -51,6 +52,7 @@ public class ContentSummaryComputationContext {
this.limitPerRun = limitPerRun;
this.nextCountLimit = limitPerRun;
this.counts = new ContentCounts.Builder().build();
+ this.snapshotCounts = new ContentCounts.Builder().build();
this.sleepMilliSec = sleepMicroSec/1000;
this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000);
}
@@ -125,6 +127,10 @@ public class ContentSummaryComputationContext {
return counts;
}
+ public ContentCounts getSnapshotCounts() {
+ return snapshotCounts;
+ }
+
public BlockStoragePolicySuite getBlockStoragePolicySuite() {
Preconditions.checkState((bsps != null || fsn != null),
"BlockStoragePolicySuite must be either initialized or available via" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 0348c1f..c6258a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -428,8 +428,9 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
*/
public final ContentSummary computeAndConvertContentSummary(int snapshotId,
ContentSummaryComputationContext summary) {
- ContentCounts counts = computeContentSummary(snapshotId, summary)
- .getCounts();
+ computeContentSummary(snapshotId, summary);
+ final ContentCounts counts = summary.getCounts();
+ final ContentCounts snapshotCounts = summary.getSnapshotCounts();
final QuotaCounts q = getQuotaCounts();
return new ContentSummary.Builder().
length(counts.getLength()).
@@ -440,6 +441,10 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
spaceQuota(q.getStorageSpace()).
typeConsumed(counts.getTypeSpaces()).
typeQuota(q.getTypeSpaces().asArray()).
+ snapshotLength(snapshotCounts.getLength()).
+ snapshotFileCount(snapshotCounts.getFileCount()).
+ snapshotDirectoryCount(snapshotCounts.getDirectoryCount()).
+ snapshotSpaceConsumed(snapshotCounts.getStoragespace()).
build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
index d00c136..9a8f9b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -634,6 +634,10 @@ public class INodeDirectory extends INodeWithAdditionalFields
// computation should include all the deleted files/directories
sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
summary.getCounts());
+ // Also compute ContentSummary for snapshotCounts (So we can extract it
+ // later from the ContentSummary of all).
+ sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
+ summary.getSnapshotCounts());
}
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0efea49/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index dcf683e..73ebf28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -113,6 +113,11 @@ public class TestDFSShell {
return p;
}
+ static void rmr(FileSystem fs, Path p) throws IOException {
+ assertTrue(fs.delete(p, true));
+ assertFalse(fs.exists(p));
+ }
+
static File createLocalFile(File f) throws IOException {
assertTrue(!f.exists());
PrintWriter out = new PrintWriter(f);
@@ -189,7 +194,7 @@ public class TestDFSShell {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
FileSystem fs = cluster.getFileSystem();
assertTrue("Not a HDFS: " + fs.getUri(),
- fs instanceof DistributedFileSystem);
+ fs instanceof DistributedFileSystem);
try {
fs.mkdirs(new Path(new Path("parent"), "child"));
try {
@@ -224,6 +229,7 @@ public class TestDFSShell {
shell.setConf(conf);
try {
+ cluster.waitActive();
Path myPath = new Path("/test/dir");
assertTrue(fs.mkdirs(myPath));
assertTrue(fs.exists(myPath));
@@ -251,7 +257,7 @@ public class TestDFSShell {
assertTrue(val == 0);
String returnString = out.toString();
out.reset();
- // Check if size matchs as expected
+ // Check if size matches as expected
assertThat(returnString, containsString(myFileLength.toString()));
assertThat(returnString, containsString(myFileDiskUsed.toString()));
assertThat(returnString, containsString(myFile2Length.toString()));
@@ -308,7 +314,232 @@ public class TestDFSShell {
System.setOut(psBackup);
cluster.shutdown();
}
+ }
+
+ @Test (timeout = 180000)
+ public void testDuSnapshots() throws IOException {
+ final int replication = 2;
+ final Configuration conf = new HdfsConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(replication).build();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final PrintStream psBackup = System.out;
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ final PrintStream psOut = new PrintStream(out);
+ final FsShell shell = new FsShell();
+ shell.setConf(conf);
+
+ try {
+ System.setOut(psOut);
+ cluster.waitActive();
+ final Path parent = new Path("/test");
+ final Path dir = new Path(parent, "dir");
+ mkdir(dfs, dir);
+ final Path file = new Path(dir, "file");
+ writeFile(dfs, file);
+ final Path file2 = new Path(dir, "file2");
+ writeFile(dfs, file2);
+ final Long fileLength = dfs.getFileStatus(file).getLen();
+ final Long fileDiskUsed = fileLength * replication;
+ final Long file2Length = dfs.getFileStatus(file2).getLen();
+ final Long file2DiskUsed = file2Length * replication;
+
+ /*
+ * Construct dir as follows:
+ * /test/dir/file <- this will later be deleted after snapshot taken.
+ * /test/dir/newfile <- this will be created after snapshot taken.
+ * /test/dir/file2
+ * Snapshot enabled on /test
+ */
+
+ // test -du on /test/dir
+ int ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", dir.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ String returnString = out.toString();
+ LOG.info("-du return is:\n" + returnString);
+ // Check if size matches as expected
+ assertTrue(returnString.contains(fileLength.toString()));
+ assertTrue(returnString.contains(fileDiskUsed.toString()));
+ assertTrue(returnString.contains(file2Length.toString()));
+ assertTrue(returnString.contains(file2DiskUsed.toString()));
+ out.reset();
+
+ // take a snapshot, then remove file and add newFile
+ final String snapshotName = "ss1";
+ final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
+ dfs.allowSnapshot(parent);
+ assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
+ rmr(dfs, file);
+ final Path newFile = new Path(dir, "newfile");
+ writeFile(dfs, newFile);
+ final Long newFileLength = dfs.getFileStatus(newFile).getLen();
+ final Long newFileDiskUsed = newFileLength * replication;
+
+ // test -du -s on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", "-s", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du -s return is:\n" + returnString);
+ Long combinedLength = fileLength + file2Length + newFileLength;
+ Long combinedDiskUsed = fileDiskUsed + file2DiskUsed + newFileDiskUsed;
+ assertTrue(returnString.contains(combinedLength.toString()));
+ assertTrue(returnString.contains(combinedDiskUsed.toString()));
+ out.reset();
+
+ // test -du on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du return is:\n" + returnString);
+ assertTrue(returnString.contains(combinedLength.toString()));
+ assertTrue(returnString.contains(combinedDiskUsed.toString()));
+ out.reset();
+
+ // test -du -s -x on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", "-s", "-x", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du -s -x return is:\n" + returnString);
+ Long exludeSnapshotLength = file2Length + newFileLength;
+ Long excludeSnapshotDiskUsed = file2DiskUsed + newFileDiskUsed;
+ assertTrue(returnString.contains(exludeSnapshotLength.toString()));
+ assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString()));
+ out.reset();
+ // test -du -x on /test
+ ret = -1;
+ try {
+ ret = shell.run(new String[] {"-du", "-x", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, ret);
+ returnString = out.toString();
+ LOG.info("-du -x return is:\n" + returnString);
+ assertTrue(returnString.contains(exludeSnapshotLength.toString()));
+ assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString()));
+ out.reset();
+ } finally {
+ System.setOut(psBackup);
+ cluster.shutdown();
+ }
+ }
+
+ @Test (timeout = 180000)
+ public void testCountSnapshots() throws IOException {
+ final int replication = 2;
+ final Configuration conf = new HdfsConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(replication).build();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final PrintStream psBackup = System.out;
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ final PrintStream psOut = new PrintStream(out);
+ System.setOut(psOut);
+ final FsShell shell = new FsShell();
+ shell.setConf(conf);
+
+ try {
+ cluster.waitActive();
+ final Path parent = new Path("/test");
+ final Path dir = new Path(parent, "dir");
+ mkdir(dfs, dir);
+ final Path file = new Path(dir, "file");
+ writeFile(dfs, file);
+ final Path file2 = new Path(dir, "file2");
+ writeFile(dfs, file2);
+ final long fileLength = dfs.getFileStatus(file).getLen();
+ final long file2Length = dfs.getFileStatus(file2).getLen();
+ final Path dir2 = new Path(parent, "dir2");
+ mkdir(dfs, dir2);
+
+ /*
+ * Construct dir as follows:
+ * /test/dir/file <- this will later be deleted after snapshot taken.
+ * /test/dir/newfile <- this will be created after snapshot taken.
+ * /test/dir/file2
+ * /test/dir2 <- this will later be deleted after snapshot taken.
+ * Snapshot enabled on /test
+ */
+
+ // take a snapshot
+ // then create /test/dir/newfile and remove /test/dir/file, /test/dir2
+ final String snapshotName = "s1";
+ final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
+ dfs.allowSnapshot(parent);
+ assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
+ rmr(dfs, file);
+ rmr(dfs, dir2);
+ final Path newFile = new Path(dir, "new file");
+ writeFile(dfs, newFile);
+ final Long newFileLength = dfs.getFileStatus(newFile).getLen();
+
+ // test -count on /test. Include header for easier debugging.
+ int val = -1;
+ try {
+ val = shell.run(new String[] {"-count", "-v", parent.toString() });
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, val);
+ String returnString = out.toString();
+ LOG.info("-count return is:\n" + returnString);
+ Scanner in = new Scanner(returnString);
+ in.nextLine();
+ assertEquals(3, in.nextLong()); //DIR_COUNT
+ assertEquals(3, in.nextLong()); //FILE_COUNT
+ assertEquals(fileLength + file2Length + newFileLength,
+ in.nextLong()); //CONTENT_SIZE
+ out.reset();
+
+ // test -count -x on /test. Include header for easier debugging.
+ val = -1;
+ try {
+ val =
+ shell.run(new String[] {"-count", "-x", "-v", parent.toString()});
+ } catch (Exception e) {
+ System.err.println("Exception raised from DFSShell.run " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, val);
+ returnString = out.toString();
+ LOG.info("-count -x return is:\n" + returnString);
+ in = new Scanner(returnString);
+ in.nextLine();
+ assertEquals(2, in.nextLong()); //DIR_COUNT
+ assertEquals(2, in.nextLong()); //FILE_COUNT
+ assertEquals(file2Length + newFileLength, in.nextLong()); //CONTENT_SIZE
+ out.reset();
+ } finally {
+ System.setOut(psBackup);
+ cluster.shutdown();
+ }
}
@Test (timeout = 30000)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/52] [abbrv] hadoop git commit: YARN-5537. Fix intermittent
failure of TestAMRMClient#testAMRMClientWithContainerResourceChange (Bibin A
Chundatt via Varun Saxena)
Posted by aw...@apache.org.
YARN-5537. Fix intermittent failure of TestAMRMClient#testAMRMClientWithContainerResourceChange (Bibin A Chundatt via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79603f58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79603f58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79603f58
Branch: refs/heads/HADOOP-13341
Commit: 79603f588238d47c77748682add0502fb89bbf48
Parents: ab3b727
Author: Varun Saxena <va...@apache.org>
Authored: Thu Aug 25 14:10:34 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Thu Aug 25 14:10:34 2016 +0530
----------------------------------------------------------------------
.../yarn/client/api/impl/TestAMRMClient.java | 47 +++++++++++---------
1 file changed, 27 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79603f58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 57cdbfb..1eeeb78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -1007,26 +1007,33 @@ public class TestAMRMClient {
Assert.assertEquals(2, amClientImpl.pendingChange.size());
// as of now: container1 asks to decrease to (512, 1)
// container2 asks to increase to (2048, 1)
- // send allocation requests
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- Assert.assertEquals(0, amClientImpl.change.size());
- // we should get decrease confirmation right away
- List<Container> decreasedContainers =
- allocResponse.getDecreasedContainers();
- List<Container> increasedContainers =
- allocResponse.getIncreasedContainers();
- Assert.assertEquals(1, decreasedContainers.size());
- Assert.assertEquals(0, increasedContainers.size());
- // we should get increase allocation after the next NM's heartbeat to RM
- sleep(150);
- // get allocations
- allocResponse = amClient.allocate(0.1f);
- decreasedContainers =
- allocResponse.getDecreasedContainers();
- increasedContainers =
- allocResponse.getIncreasedContainers();
- Assert.assertEquals(1, increasedContainers.size());
- Assert.assertEquals(0, decreasedContainers.size());
+ List<Container> decreasedContainers;
+ List<Container> increasedContainers;
+ int allocateAttempts = 0;
+ int decreased = 0;
+ int increased = 0;
+ while (allocateAttempts < 30) {
+ // send allocation requests
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ decreasedContainers = allocResponse.getDecreasedContainers();
+ increasedContainers = allocResponse.getIncreasedContainers();
+ decreased += decreasedContainers.size();
+ increased += increasedContainers.size();
+ if (allocateAttempts == 0) {
+ // we should get decrease confirmation right away
+ Assert.assertEquals(1, decreased);
+ // After first allocate request check change size
+ Assert.assertEquals(0, amClientImpl.change.size());
+ } else if (increased == 1) {
+ break;
+ }
+ // increase request is served after next NM heart beat is received
+ // Sleeping and retrying allocate
+ sleep(20);
+ allocateAttempts++;
+ }
+ Assert.assertEquals(1, decreased);
+ Assert.assertEquals(1, increased);
}
private void testAllocation(final AMRMClientImpl<ContainerRequest> amClient)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/52] [abbrv] hadoop git commit: HDFS-9745.
TestSecureNNWithQJM#testSecureMode sometimes fails with timeouts. Contributed
by Xiao Chen
Posted by aw...@apache.org.
HDFS-9745. TestSecureNNWithQJM#testSecureMode sometimes fails with timeouts. Contributed by Xiao Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/126d165e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/126d165e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/126d165e
Branch: refs/heads/HADOOP-13341
Commit: 126d165efd80e266a8309241f3cf059e358f5019
Parents: 6f9c346
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Aug 23 14:46:58 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Aug 23 14:46:58 2016 +0000
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/126d165e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java
index 9abfb9c..46b016f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/TestSecureNNWithQJM.java
@@ -79,7 +79,7 @@ public class TestSecureNNWithQJM {
private MiniJournalCluster mjc;
@Rule
- public Timeout timeout = new Timeout(30000);
+ public Timeout timeout = new Timeout(180000);
@BeforeClass
public static void init() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/52] [abbrv] hadoop git commit: YARN-5389.
TestYarnClient#testReservationDelete fails. Contributed by Sean Po
Posted by aw...@apache.org.
YARN-5389. TestYarnClient#testReservationDelete fails. Contributed by Sean Po
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d86110a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d86110a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d86110a
Branch: refs/heads/HADOOP-13341
Commit: 3d86110a5ccfdaff8671fb6ad8f67b4ab66f33da
Parents: 52b25fe
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Aug 25 14:42:06 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Aug 25 14:42:06 2016 +0000
----------------------------------------------------------------------
.../yarn/client/api/impl/TestYarnClient.java | 43 +++++++++-----------
1 file changed, 19 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d86110a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 360ff99..dd19acb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -41,6 +41,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.base.Supplier;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputByteBuffer;
@@ -52,6 +53,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
@@ -1193,7 +1195,7 @@ public class TestYarnClient {
}
}
- private MiniYARNCluster setupMiniYARNCluster() {
+ private MiniYARNCluster setupMiniYARNCluster() throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
ReservationSystemTestUtil.setupQueueConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
@@ -1205,23 +1207,15 @@ public class TestYarnClient {
cluster.init(conf);
cluster.start();
- int attempts;
- for (attempts = 10; attempts > 0; attempts--) {
- if (cluster.getResourceManager().getRMContext().getReservationSystem()
- .getPlan(ReservationSystemTestUtil.reservationQ).getTotalCapacity()
- .getMemorySize() > 6000) {
- break;
- }
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- e.printStackTrace();
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return cluster.getResourceManager().getRMContext()
+ .getReservationSystem()
+ .getPlan(ReservationSystemTestUtil.reservationQ)
+ .getTotalCapacity().getMemorySize() > 6000;
}
- }
- if (attempts <= 0) {
- Assert.fail("Exhausted attempts in checking if node capacity was "
- + "added to the plan");
- }
+ }, 10, 10000);
return cluster;
}
@@ -1253,7 +1247,7 @@ public class TestYarnClient {
}
@Test
- public void testCreateReservation() {
+ public void testCreateReservation() throws Exception {
MiniYARNCluster cluster = setupMiniYARNCluster();
YarnClient client = setupYarnClient(cluster);
try {
@@ -1296,7 +1290,7 @@ public class TestYarnClient {
}
@Test
- public void testUpdateReservation() {
+ public void testUpdateReservation() throws Exception {
MiniYARNCluster cluster = setupMiniYARNCluster();
YarnClient client = setupYarnClient(cluster);
try {
@@ -1338,7 +1332,7 @@ public class TestYarnClient {
}
@Test
- public void testListReservationsByReservationId() {
+ public void testListReservationsByReservationId() throws Exception{
MiniYARNCluster cluster = setupMiniYARNCluster();
YarnClient client = setupYarnClient(cluster);
try {
@@ -1375,7 +1369,7 @@ public class TestYarnClient {
}
@Test
- public void testListReservationsByTimeInterval() {
+ public void testListReservationsByTimeInterval() throws Exception {
MiniYARNCluster cluster = setupMiniYARNCluster();
YarnClient client = setupYarnClient(cluster);
try {
@@ -1440,7 +1434,7 @@ public class TestYarnClient {
}
@Test
- public void testListReservationsByInvalidTimeInterval() {
+ public void testListReservationsByInvalidTimeInterval() throws Exception {
MiniYARNCluster cluster = setupMiniYARNCluster();
YarnClient client = setupYarnClient(cluster);
try {
@@ -1490,7 +1484,8 @@ public class TestYarnClient {
}
@Test
- public void testListReservationsByTimeIntervalContainingNoReservations() {
+ public void testListReservationsByTimeIntervalContainingNoReservations()
+ throws Exception {
MiniYARNCluster cluster = setupMiniYARNCluster();
YarnClient client = setupYarnClient(cluster);
try {
@@ -1580,7 +1575,7 @@ public class TestYarnClient {
}
@Test
- public void testReservationDelete() {
+ public void testReservationDelete() throws Exception {
MiniYARNCluster cluster = setupMiniYARNCluster();
YarnClient client = setupYarnClient(cluster);
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/52] [abbrv] hadoop git commit: MAPREDUCE-6762.
ControlledJob#toString failed with NPE when job status is not successfully
updated (Weiwei Yang via Varun Saxena)
Posted by aw...@apache.org.
MAPREDUCE-6762. ControlledJob#toString failed with NPE when job status is not successfully updated (Weiwei Yang via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d37b45d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d37b45d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d37b45d6
Branch: refs/heads/HADOOP-13341
Commit: d37b45d613b768950d1cbe342961cd71776816ae
Parents: 0faee62
Author: Varun Saxena <va...@apache.org>
Authored: Sun Aug 21 21:46:17 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Sun Aug 21 21:46:17 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapreduce/Job.java | 2 +-
.../org/apache/hadoop/mapreduce/TestJob.java | 36 ++++++++++++++++++++
2 files changed, 37 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37b45d6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
index 33e820b..45c065d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
@@ -422,7 +422,7 @@ public class Job extends JobContextImpl implements JobContext {
* The user-specified job name.
*/
public String getJobName() {
- if (state == JobState.DEFINE) {
+ if (state == JobState.DEFINE || status == null) {
return super.getJobName();
}
ensureState(JobState.RUNNING);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37b45d6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
index 71bacf7..60f390f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.JobStatus.State;
+import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -55,6 +56,41 @@ public class TestJob {
}
@Test
+ public void testUnexpectedJobStatus() throws Exception {
+ Cluster cluster = mock(Cluster.class);
+ JobID jobid = new JobID("1014873536921", 6);
+ ClientProtocol clientProtocol = mock(ClientProtocol.class);
+ when(cluster.getClient()).thenReturn(clientProtocol);
+ JobStatus status = new JobStatus(jobid, 0f, 0f, 0f, 0f,
+ State.RUNNING, JobPriority.DEFAULT, "root",
+ "testUnexpectedJobStatus", "job file", "tracking URL");
+ when(clientProtocol.getJobStatus(jobid)).thenReturn(status);
+ Job job = Job.getInstance(cluster, status, new JobConf());
+
+ // ensurer job status is RUNNING
+ Assert.assertNotNull(job.getStatus());
+ Assert.assertTrue(job.getStatus().getState() == State.RUNNING);
+
+ // when updating job status, job client could not retrieve
+ // job status, and status reset to null
+ when(clientProtocol.getJobStatus(jobid)).thenReturn(null);
+
+ try {
+ job.updateStatus();
+ } catch (IOException e) {
+ Assert.assertTrue(e != null
+ && e.getMessage().contains("Job status not available"));
+ }
+
+ try {
+ ControlledJob cj = new ControlledJob(job, null);
+ Assert.assertNotNull(cj.toString());
+ } catch (NullPointerException e) {
+ Assert.fail("job API fails with NPE");
+ }
+ }
+
+ @Test
public void testUGICredentialsPropogation() throws Exception {
Credentials creds = new Credentials();
Token<?> token = mock(Token.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/52] [abbrv] hadoop git commit: HADOOP-13534. Remove unused
TrashPolicy#getInstance and initialize code. Contributed by Yiqun Lin.
Posted by aw...@apache.org.
HADOOP-13534. Remove unused TrashPolicy#getInstance and initialize code. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab3b727b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab3b727b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab3b727b
Branch: refs/heads/HADOOP-13341
Commit: ab3b727b5f1511ea05f75d8798eaf85f6defcf53
Parents: 5a6fc5f
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Aug 25 13:54:57 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Aug 25 13:54:57 2016 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/TrashPolicy.java | 30 --------------------
.../apache/hadoop/fs/TrashPolicyDefault.java | 15 ----------
.../java/org/apache/hadoop/fs/TestTrash.java | 4 ---
3 files changed, 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3b727b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
index 157b9ab..bd99db4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
@@ -38,17 +38,6 @@ public abstract class TrashPolicy extends Configured {
/**
* Used to setup the trash policy. Must be implemented by all TrashPolicy
- * implementations.
- * @param conf the configuration to be used
- * @param fs the filesystem to be used
- * @param home the home directory
- * @deprecated Use {@link #initialize(Configuration, FileSystem)} instead.
- */
- @Deprecated
- public abstract void initialize(Configuration conf, FileSystem fs, Path home);
-
- /**
- * Used to setup the trash policy. Must be implemented by all TrashPolicy
* implementations. Different from initialize(conf, fs, home), this one does
* not assume trash always under /user/$USER due to HDFS encryption zone.
* @param conf the configuration to be used
@@ -116,25 +105,6 @@ public abstract class TrashPolicy extends Configured {
*
* @param conf the configuration to be used
* @param fs the file system to be used
- * @param home the home directory
- * @return an instance of TrashPolicy
- * @deprecated Use {@link #getInstance(Configuration, FileSystem)} instead.
- */
- @Deprecated
- public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) {
- Class<? extends TrashPolicy> trashClass = conf.getClass(
- "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class);
- TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf);
- trash.initialize(conf, fs, home); // initialize TrashPolicy
- return trash;
- }
-
- /**
- * Get an instance of the configured TrashPolicy based on the value
- * of the configuration parameter fs.trash.classname.
- *
- * @param conf the configuration to be used
- * @param fs the file system to be used
* @return an instance of TrashPolicy
*/
public static TrashPolicy getInstance(Configuration conf, FileSystem fs)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3b727b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index 72222be..f4a825c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -75,21 +75,6 @@ public class TrashPolicyDefault extends TrashPolicy {
initialize(conf, fs);
}
- /**
- * @deprecated Use {@link #initialize(Configuration, FileSystem)} instead.
- */
- @Override
- @Deprecated
- public void initialize(Configuration conf, FileSystem fs, Path home) {
- this.fs = fs;
- this.deletionInterval = (long)(conf.getFloat(
- FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT)
- * MSECS_PER_MINUTE);
- this.emptierInterval = (long)(conf.getFloat(
- FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT)
- * MSECS_PER_MINUTE);
- }
-
@Override
public void initialize(Configuration conf, FileSystem fs) {
this.fs = fs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3b727b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 338aff6..2aba01f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -692,10 +692,6 @@ public class TestTrash extends TestCase {
public TestTrashPolicy() { }
@Override
- public void initialize(Configuration conf, FileSystem fs, Path home) {
- }
-
- @Override
public void initialize(Configuration conf, FileSystem fs) {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/52] [abbrv] hadoop git commit: YARN-5544.
TestNodeBlacklistingOnAMFailures fails on trunk. Contributed by Sunil G.
Posted by aw...@apache.org.
YARN-5544. TestNodeBlacklistingOnAMFailures fails on trunk. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d5997d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d5997d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d5997d2
Branch: refs/heads/HADOOP-13341
Commit: 0d5997d2b98eb89e72828dfcd78f02aa4e7e1e67
Parents: 8cc4a67
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Aug 23 10:33:28 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Aug 23 14:37:39 2016 +0530
----------------------------------------------------------------------
.../TestNodeBlacklistingOnAMFailures.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d5997d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
index ef6d43b..7a24b7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java
@@ -68,6 +68,9 @@ public class TestNodeBlacklistingOnAMFailures {
MockRM rm = startRM(conf, dispatcher);
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+ // Register 5 nodes, so that we can blacklist atleast one if AM container
+ // is failed. As per calculation it will be like, 5nodes * 0.2 (default)=1.
+ // First register 2 nodes, and after AM lauched register 3 more nodes.
MockNM nm1 =
new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService());
nm1.registerNode();
@@ -93,6 +96,19 @@ public class TestNodeBlacklistingOnAMFailures {
otherNode = nm1;
}
+ // register 3 nodes now
+ MockNM nm3 =
+ new MockNM("127.0.0.3:2345", 8000, rm.getResourceTrackerService());
+ nm3.registerNode();
+
+ MockNM nm4 =
+ new MockNM("127.0.0.4:2345", 8000, rm.getResourceTrackerService());
+ nm4.registerNode();
+
+ MockNM nm5 =
+ new MockNM("127.0.0.5:2345", 8000, rm.getResourceTrackerService());
+ nm5.registerNode();
+
// Set the exist status to INVALID so that we can verify that the system
// automatically blacklisting the node
makeAMContainerExit(rm, amContainerId, currentNode,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/52] [abbrv] hadoop git commit: MAPREDUCE-6578. Add support for
HDFS heterogeneous storage testing to TestDFSIO. Contributed by Wei Zhou and
Sammi Chen
Posted by aw...@apache.org.
MAPREDUCE-6578. Add support for HDFS heterogeneous storage testing to TestDFSIO. Contributed by Wei Zhou and Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ce1ab95
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ce1ab95
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ce1ab95
Branch: refs/heads/HADOOP-13341
Commit: 0ce1ab95cc1178f9ea763fd1f5a65a890b23b0de
Parents: 793447f
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Aug 24 22:17:05 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Aug 24 22:17:05 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/TestDFSIO.java | 53 +++++++++++++++++---
1 file changed, 47 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ce1ab95/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
index e7aa66b..05d4d77 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
@@ -29,6 +29,7 @@ import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.text.DecimalFormat;
+import java.util.Collection;
import java.util.Date;
import java.util.Random;
import java.util.StringTokenizer;
@@ -36,7 +37,9 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
@@ -102,7 +105,8 @@ public class TestDFSIO implements Tool {
" [-compression codecClassName]" +
" [-nrFiles N]" +
" [-size Size[B|KB|MB|GB|TB]]" +
- " [-resFile resultFileName] [-bufferSize Bytes]";
+ " [-resFile resultFileName] [-bufferSize Bytes]" +
+ " [-storagePolicy storagePolicyName]";
private Configuration config;
@@ -305,7 +309,7 @@ public class TestDFSIO implements Tool {
writer = null;
}
}
- LOG.info("created control files for: "+nrFiles+" files");
+ LOG.info("created control files for: " + nrFiles + " files");
}
private static String getFileName(int fIdx) {
@@ -326,6 +330,7 @@ public class TestDFSIO implements Tool {
*/
private abstract static class IOStatMapper extends IOMapperBase<Long> {
protected CompressionCodec compressionCodec;
+ protected String blockStoragePolicy;
IOStatMapper() {
}
@@ -350,6 +355,8 @@ public class TestDFSIO implements Tool {
compressionCodec = (CompressionCodec)
ReflectionUtils.newInstance(codec, getConf());
}
+
+ blockStoragePolicy = getConf().get("test.io.block.storage.policy", null);
}
@Override // IOMapperBase
@@ -389,8 +396,11 @@ public class TestDFSIO implements Tool {
@Override // IOMapperBase
public Closeable getIOStream(String name) throws IOException {
// create file
- OutputStream out =
- fs.create(new Path(getDataDir(getConf()), name), true, bufferSize);
+ Path filePath = new Path(getDataDir(getConf()), name);
+ OutputStream out = fs.create(filePath, true, bufferSize);
+ if (blockStoragePolicy != null) {
+ fs.setStoragePolicy(filePath, blockStoragePolicy);
+ }
if(compressionCodec != null)
out = compressionCodec.createOutputStream(out);
LOG.info("out = " + out.getClass().getName());
@@ -713,8 +723,9 @@ public class TestDFSIO implements Tool {
System.err.print(StringUtils.stringifyException(e));
res = -2;
}
- if(res == -1)
- System.err.print(USAGE);
+ if (res == -1) {
+ System.err.println(USAGE);
+ }
System.exit(res);
}
@@ -727,6 +738,7 @@ public class TestDFSIO implements Tool {
long skipSize = 0;
String resFileName = DEFAULT_RES_FILE_NAME;
String compressionClass = null;
+ String storagePolicy = null;
boolean isSequential = false;
String version = TestDFSIO.class.getSimpleName() + ".1.8";
@@ -771,6 +783,8 @@ public class TestDFSIO implements Tool {
bufferSize = Integer.parseInt(args[++i]);
} else if (args[i].equalsIgnoreCase("-resfile")) {
resFileName = args[++i];
+ } else if (args[i].equalsIgnoreCase("-storagePolicy")) {
+ storagePolicy = args[++i];
} else {
System.err.println("Illegal argument: " + args[i]);
return -1;
@@ -799,6 +813,33 @@ public class TestDFSIO implements Tool {
config.setLong("test.io.skip.size", skipSize);
FileSystem fs = FileSystem.get(config);
+ if (storagePolicy != null) {
+ boolean isValid = false;
+ Collection<BlockStoragePolicy> storagePolicies =
+ ((DistributedFileSystem) fs).getAllStoragePolicies();
+ try {
+ for (BlockStoragePolicy policy : storagePolicies) {
+ if (policy.getName().equals(storagePolicy)) {
+ isValid = true;
+ break;
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException("Get block storage policies error: ", e);
+ }
+ if (!isValid) {
+ System.out.println("Invalid block storage policy: " + storagePolicy);
+ System.out.println("Current supported storage policy list: ");
+ for (BlockStoragePolicy policy : storagePolicies) {
+ System.out.println(policy.getName());
+ }
+ return -1;
+ }
+
+ config.set("test.io.block.storage.policy", storagePolicy);
+ LOG.info("storagePolicy = " + storagePolicy);
+ }
+
if (isSequential) {
long tStart = System.currentTimeMillis();
sequentialTest(fs, testType, nrBytes, nrFiles);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/52] [abbrv] hadoop git commit: YARN-5042. Mount /sys/fs/cgroup
into Docker containers as read only mount. Contributed by luhuichun.
Posted by aw...@apache.org.
YARN-5042. Mount /sys/fs/cgroup into Docker containers as read only mount. Contributed by luhuichun.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/525d52bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/525d52bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/525d52bf
Branch: refs/heads/HADOOP-13341
Commit: 525d52bf7c18d1e334e02e6fb936400c9cf2b0bc
Parents: 79603f5
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Aug 25 12:56:11 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu Aug 25 14:18:26 2016 +0530
----------------------------------------------------------------------
.../linux/runtime/DockerLinuxContainerRuntime.java | 7 ++++---
.../linux/runtime/docker/DockerRunCommand.java | 7 ++++++-
.../linux/runtime/TestDockerContainerRuntime.java | 4 ++++
3 files changed, 14 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/525d52bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index be17af9..2cce1f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -447,7 +447,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.detachOnRun()
.setContainerWorkDir(containerWorkDir.toString())
.setNetworkType(network)
- .setCapabilities(capabilities);
+ .setCapabilities(capabilities)
+ .addMountLocation("/sys/fs/cgroup", "/sys/fs/cgroup:ro", false);
List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.addAll(filecacheDirs);
@@ -455,7 +456,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
allDirs.addAll(containerLogDirs);
allDirs.addAll(userLocalDirs);
for (String dir: allDirs) {
- runCommand.addMountLocation(dir, dir);
+ runCommand.addMountLocation(dir, dir, true);
}
if (environment.containsKey(ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS)) {
@@ -470,7 +471,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
String src = validateMount(dir[0], localizedResources);
String dst = dir[1];
- runCommand.addMountLocation(src, dst + ":ro");
+ runCommand.addMountLocation(src, dst + ":ro", true);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/525d52bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index 7c49ef9..f79f4ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -22,6 +22,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime
import org.apache.hadoop.util.StringUtils;
+import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -59,7 +60,11 @@ public class DockerRunCommand extends DockerCommand {
}
public DockerRunCommand addMountLocation(String sourcePath, String
- destinationPath) {
+ destinationPath, boolean createSource) {
+ boolean sourceExists = new File(sourcePath).exists();
+ if (!sourceExists && !createSource) {
+ return this;
+ }
super.addCommandArguments("-v", sourcePath + ":" + destinationPath);
return this;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/525d52bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index ad8b52d..a05ff46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -276,6 +276,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=host ")
.append(getExpectedTestCapabilitiesArgumentString())
+ .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
.append("-v %4$s:%4$s ")
.append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ")
@@ -381,6 +382,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + allowedNetwork + " ")
.append(getExpectedTestCapabilitiesArgumentString())
+ .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
@@ -435,6 +437,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + customNetwork1 + " ")
.append(getExpectedTestCapabilitiesArgumentString())
+ .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
@@ -471,6 +474,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + customNetwork2 + " ")
.append(getExpectedTestCapabilitiesArgumentString())
+ .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/52] [abbrv] hadoop git commit: HADOOP-13538. Deprecate
getInstance and initialize methods with Path in TrashPolicy. Contributed by
Yiqun Lin.
Posted by aw...@apache.org.
HADOOP-13538. Deprecate getInstance and initialize methods with Path in TrashPolicy. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/092b4d5b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/092b4d5b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/092b4d5b
Branch: refs/heads/HADOOP-13341
Commit: 092b4d5bfd02131d62723fc5673892305eb9fcef
Parents: c37346d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Aug 24 16:47:52 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Aug 24 16:47:52 2016 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/TrashPolicy.java | 4 ++++
.../src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java | 4 ++++
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/092b4d5b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
index 92a4d1f..157b9ab 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
@@ -42,7 +42,9 @@ public abstract class TrashPolicy extends Configured {
* @param conf the configuration to be used
* @param fs the filesystem to be used
* @param home the home directory
+ * @deprecated Use {@link #initialize(Configuration, FileSystem)} instead.
*/
+ @Deprecated
public abstract void initialize(Configuration conf, FileSystem fs, Path home);
/**
@@ -116,7 +118,9 @@ public abstract class TrashPolicy extends Configured {
* @param fs the file system to be used
* @param home the home directory
* @return an instance of TrashPolicy
+ * @deprecated Use {@link #getInstance(Configuration, FileSystem)} instead.
*/
+ @Deprecated
public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) {
Class<? extends TrashPolicy> trashClass = conf.getClass(
"fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/092b4d5b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index 66ef890..72222be 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -75,7 +75,11 @@ public class TrashPolicyDefault extends TrashPolicy {
initialize(conf, fs);
}
+ /**
+ * @deprecated Use {@link #initialize(Configuration, FileSystem)} instead.
+ */
@Override
+ @Deprecated
public void initialize(Configuration conf, FileSystem fs, Path home) {
this.fs = fs;
this.deletionInterval = (long)(conf.getFloat(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/52] [abbrv] hadoop git commit: HDFS-10692. Update JDiff report's
base version for HDFS from 2.6.0 to 2.7.2. Contributed by Wangda Tan.
Posted by aw...@apache.org.
HDFS-10692. Update JDiff report's base version for HDFS from 2.6.0 to 2.7.2. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc7a1c54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc7a1c54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc7a1c54
Branch: refs/heads/HADOOP-13341
Commit: dc7a1c54f4447fb01980ae61d5d67e90c4f52f00
Parents: 115ecb5
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Mon Aug 22 11:35:18 2016 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Mon Aug 22 11:35:18 2016 -0700
----------------------------------------------------------------------
.../jdiff/Apache_Hadoop_HDFS_2.7.2.xml | 21505 +++++++++++++++++
1 file changed, 21505 insertions(+)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org